{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bd0160b8-de87-4a9c-bfe2-b678e89cde89",
   "metadata": {},
   "source": [
    "Compare the TransformerLens implementation of a model to the Huggingface implementation. This script was originally use in https://github.com/TransformerLensOrg/TransformerLens/issues/570 to debug Mixtral."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e1c21b4-5a82-4838-ae2a-0c7be2708b65",
   "metadata": {},
   "source": [
    "## setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4fb7e0bc-4ef5-40c8-8222-336e83bd6e66",
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install transformers matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cba8adb4-03a4-4061-b62b-18bcc091b8af",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Everything can be configured here\n",
    "model_id = \"\"\n",
    "text = \"Hello my name is\"\n",
    "device=\"cpu\"\n",
    "# Set this to true to trigger hugging face login if needed\n",
    "gated_model = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "be241e96-3bbb-46a4-a4d4-0213eb094d6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# If you need a specific head, uncomment this and specify the head\n",
    "# %pip install git+https://github.com/TransformerLensOrg/TransformerLens.git@head\n",
    "# Otherwise, for running this on the latest release\n",
    "%pip install transformer_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6d7341d8-881c-41c3-8199-ae9590d51a5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "if gated_model:\n",
    "    %pip install huggingface_hub\n",
    "    from huggingface_hub import login\n",
    "    login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ec6d8055-dcdd-4db9-b13a-28860292ad47",
   "metadata": {},
   "outputs": [],
   "source": [
    "import einops\n",
    "from torch.testing import assert_close\n",
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "from transformer_lens import HookedTransformer\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ceea776-25d6-44b3-99e6-f38c30064954",
   "metadata": {},
   "source": [
    "## TransformerLens model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2c3cb338-cf1b-4775-b278-302999164e6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f92d32c0f474ad5a907559e872b7b7f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/19 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded pretrained model mistralai/Mixtral-8x7B-v0.1 into HookedTransformer\n"
     ]
    }
   ],
   "source": [
    "tl_model = HookedTransformer.from_pretrained_no_processing(\n",
    "    model_id,\n",
    "    device=device,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "617609fe-060e-48b5-9daf-cb45cb6cc5d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Hello my name is Kojo Akoto Boateng.  I was born in Accra at the Korle Bu Teaching Hospital.  I am the first born of Akoto Boateng.\\n\\nOn 23rd December, l was enrolled'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.generate(\n",
    "    text,\n",
    "    verbose=False,\n",
    "    max_new_tokens=50,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c36ce972-7ee4-4562-bbdb-544cba8dee16",
   "metadata": {},
   "source": [
    "## Huggingface Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4f3ccaf0-1650-4621-84f4-dbcdef132447",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c72da9eebb7450699f672f969b945f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/19 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
    "hf_model = AutoModelForCausalLM.from_pretrained(model_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "97cec596-b5fc-48c2-82db-08af7e238a0b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello my name is Katie and I am a 20 year old student from the UK. I am currently studying a degree in English Literature and Creative Writing at the University of Winchester. I have always had a passion for writing and I am hoping to pursue\n"
     ]
    }
   ],
   "source": [
    "inputs = tokenizer(text, return_tensors=\"pt\")\n",
    "outputs = hf_model.generate(**inputs, max_new_tokens=50)\n",
    "print(tokenizer.decode(outputs[0], skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5492ef7-4dc4-4a1b-ab24-9ab36e084223",
   "metadata": {},
   "source": [
    "## Compare Model Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cfa7d1c3-72e8-45b3-850c-091669f48b1e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    einops.rearrange(tl_model.blocks[0].attn.W_Q, \"n m h -> (n h) m\") ==\n",
    "    hf_model.model.layers[0].self_attn.q_proj.weight\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "83649934-f06b-4f94-8004-59b8d4098589",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([32, 4096, 128]), torch.Size([1024, 4096]))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].attn.W_K.shape, hf_model.model.layers[0].self_attn.k_proj.weight.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4fa20cf5-b720-4946-a7e5-e1d2e6277f6c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    einops.reduce(\n",
    "        tl_model.blocks[0].attn.W_K, \"(n repeat) m h -> (n h) m\",\n",
    "        'max',\n",
    "        n=tl_model.cfg.n_key_value_heads,\n",
    "        repeat=4) ==\n",
    "    hf_model.model.layers[0].self_attn.k_proj.weight\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ef6f7ea9-ef0b-4091-8d00-504b481fc59a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    einops.reduce(\n",
    "        tl_model.blocks[0].attn.W_V, \"(n repeat) m h -> (n h) m\",\n",
    "        'max',\n",
    "        n=tl_model.cfg.n_key_value_heads,\n",
    "        repeat=4) ==\n",
    "    hf_model.model.layers[0].self_attn.v_proj.weight\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "04b8f4be-ce7d-4dc2-acda-d023c721525c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    einops.rearrange(tl_model.blocks[0].attn.W_O, \"n h m -> m (n h)\") ==\n",
    "    hf_model.model.layers[0].self_attn.o_proj.weight\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1e10ed87-31b5-4c1c-b726-7a3f49fbd136",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.]], requires_grad=True)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].attn.b_Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6caf9d98-adb2-45e7-8357-34288b2156f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(hf_model.model.layers[0].block_sparse_moe.gate.weight.T == tl_model.blocks[0].mlp.W_gate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "00e9ea5d-74c2-4c2a-8e9d-6fc196cb8fc3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.float32, torch.float32)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].block_sparse_moe.gate.weight.dtype, tl_model.blocks[0].mlp.W_gate.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df857ae9-8cae-438b-941a-f5050709953e",
   "metadata": {},
   "source": [
    "## Compare Layer Outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "06e11c39-786a-4b09-8f5e-18a558259fb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_tensor = torch.randn((1, 1, 4096,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f0186768-d8f0-4d55-a94c-606c4ba3f7ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[[ 0.3073,  0.6716, -1.5622,  ...,  0.1159,  0.7766, -0.2877]]],\n",
       "        grad_fn=<AddBackward0>),)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0](test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "432bb274-b499-44c9-98d1-777d03425daa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.3073,  0.6716, -1.5622,  ...,  0.1159,  0.7766, -0.2877]]],\n",
       "       grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0](test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4a440811-e7f0-4092-b8e7-f7cac80dc84a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[True, True, True,  ..., True, True, True]]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0](test_tensor)[0] == tl_model.blocks[0](test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "8ed65bb3-6990-48e5-9ef2-1becd9dfaffc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7765660285949707, 0.7765660285949707)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0](test_tensor)[0][0, 0, -2].item(), tl_model.blocks[0](test_tensor)[0, 0, -2].item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "763f6c2e-b71f-4724-b2f7-f79a9ab29caf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(3153)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.sum(hf_model.model.layers[0](test_tensor)[0] == tl_model.blocks[0](test_tensor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f41aa0eb-6386-476d-ae1a-b5e7e06893aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAHFCAYAAAAT5Oa6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABJxElEQVR4nO3df3zP9eL///ur7bWfZjazX5lZQsoqqZh+IMzI7xM6OouSdEg03nXUu0yJkwrnzEmOGFFR50S/tEyhNPKjo+J4S/Ij2Ux+zO/tZR7fP3z3/HjZD8zsNXverpfLLhfPx/Pxejwfj8frOe4ez+fz9XIYY4wAAABs7CpPdwAAAMDTCEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CES4Ys2ePVsOh8P68fPzU2RkpNq2basJEyYoNze32GtSU1PlcDjcygoKCvTYY48pKipKXl5euvnmmyVJBw4c0P3336/w8HA5HA716NGjEkZ15dqxY4fb++FwOFSzZk3ddNNNmjJligoLC8vV7uLFi5Wamlqxna1iHA6HHn/8cU93o0Jt3rxZAwYMUL169eTj46OwsDB17txZn3322SW1+/rrr2v27NkV08nzOH78uFJTU7V8+fJKOR48y9vTHQAuVXp6uq677jq5XC7l5uZq5cqVevnll/Xqq69qwYIFat++vVX3kUceUVJSktvrp02bpunTpystLU3NmzdXjRo1JEkvvviiFi5cqFmzZqlBgwYKDQ2t1HFdqYYNG6Z+/fpJkg4dOqSPPvpITz75pH799Ve99tprF93e4sWL9Y9//KPah6Lq5IMPPlC/fv10zTXX6LnnnlPjxo21d+9epaenq3Pnzvqf//kfTZw4sVxtv/766woLC9OAAQMqttMlOH78uMaOHStJatOmzWU/HjyLQIQrXtOmTXXrrbda23/4wx/05JNP6s4771SvXr20detWRURESJLq1q2runXrur1+48aN8vf3L/Y/9I0bN6pBgwZ64IEHKqyvJ06ckL+/f4W1VxXVq1dPLVu2tLaTkpK0ceNGvfvuu+UKRKh6jh8/roCAgBL3bdu2TcnJyYqPj9fy5csVGBho7evdu7f+/Oc/65VXXtEtt9yi+++/v7K6DJwXl8xQLdWrV0+vvfaajhw5ounTp1vl514yczgcevPNN3XixAnrMk/RpbilS5dq8+bNVnnRsnlBQYHGjRun6667Tr6+vqpTp44eeugh7du3z60P9evXV5cuXfTBBx+oWbNm8vPzs/63mZOTo8GDB6tu3bry8fFRXFycxo4dq1OnTlmvL7oE9eqrr2rSpEmKi4tTjRo1lJCQoNWrVxcb87fffquuXbuqdu3a8vPzU4MGDTRixAi3Olu3blW/fv0UHh4uX19fNWnSRP/4xz/c6pw+fVrjxo1T48aN5e/vr1q1aunGG2/U3/72t3K9F5IUHBwsp9NZrHzBggVKSEhQYGCgatSooY4dO+o///mPtX/AgAFW/86+FLdjxw717t1bN9xwg1t7Xbt2lcPh0Pvvv2+Vfffdd3I4HPr444+tsguZf+ni3+uMjAzdcsst8vf313XXXadZs2aVe85KmqvExERFRUXJ399fTZo00V/+8hcdO3bMqjN37lw5HA6tWrWq2OtfeOEFOZ1O7dmzxypbunSp2rVrp5o1ayogIEB33HGHvvjiC7fXFf3OfPfdd7rvvvsUEhKiBg0alNrPyZMn6/jx40pLS3MLQ0Vee+011apVSy+99FKxY5yr6Hdxx44dks7M86ZNm7RixQrrXKhfv74kafny5XI4HJo3b55SUlIUGRkpf39/tW7d2u2cks6s9pS04jNgwACrvR07dqhOnTqSpLFjx1rHK1qZ2rdvnx599FHFxMRY58Ydd9yhpUuXljo3qOIMcIVKT083kszatWtL3H/06FHj5eVl2rVrZ5WNGTPGnH3ar1q1ynTu3Nn4+/ubVatWmVWrVpmcnByzatUq06xZM3PNNddY5Xl5eaawsNAkJSWZwMBAM3bsWJOZmWnefPNNc/XVV5vrr7/eHD9+3Go7NjbWREVFmWuuucbMmjXLLFu2zKxZs8ZkZ2ebmJgYExsba6ZPn26WLl1qXnzxRePr62sGDBhgvX779u1Gkqlfv75JSkoyixYtMosWLTLx8fEmJCTEHDp0yKqbkZFhnE6nufHGG83s2bPNl19+aWbNmmXuv/9+q86mTZtMcHCwiY+PN2+99ZZZsmSJGTlypLnqqqtMamqqVW/ChAnGy8vLjBkzxnzxxRcmIyPDTJkyxa1OSYr6+/LLLxuXy2VcLpf5/fffzcyZM423t7d59tln3eq/9NJLxuFwmIcffth88skn5oMPPjAJCQkmMDDQbNq0yRhjzM8//2zuu+8+I8l6H1atWmVOnjxp3njjDSPJ7NmzxxhjjMvlMkFBQcbf398MGjTIOs7LL79svL29zeHDh40x5oLn/2Lf67p165rrr7/evPXWW+bzzz83vXv3NpLMihUrypw3Y4yRZIYOHVpmnRdffNFMnjzZfPrpp2b58uXmjTfeMHFxcaZt27ZWnfz8fBMZGWkeeOABt9e6XC4THR1tevfubZXNnTvXOBwO06NHD/PBBx+Yjz/+2HTp0sV4eXmZpUuXWvWKfmdiY2PN008/bTIzM82iRYtK7WejRo1MREREmWPp06ePkWSys7PdjnGuot/x7du3G2OM+e6778w111xjmjVrZp0L3333nTHGmGXLlhlJJiYmxnTv3t18/PHHZt68eebaa681NWvWNNu2bbPabd26tWndunWx4/Xv39/ExsYaY4w5efKkycjIMJLMwIEDreP9/PPPxhhjOnbsaOrUqWP++c9/muXLl5tFixaZ559/3syfP7/MsaPqIhDhinW+QGSMMREREaZJkybWdkl/8fbv398EBgYWe23r1q3NDTfc4Fb27rvvGknm3//+t1v52rVrjSTz+uuvW2WxsbHGy8vLbNmyxa3u4MGDTY0aNczOnTvdyl999VUjyQoDRQEjPj7enDp1yqq3Zs0aI8m8++67VlmDBg1MgwYNzIkTJ0qdi44dO5q6deuavLw8t/LHH3/c+Pn5mQMHDhhjjOnSpYu5+eabS22nNEX9LelnwIABbmPYtWuX8fb2NsOGDXNr48iRIyYyMtL06dPHKhs6dGiJ/1j+/PPPRpJ56623jDHGrFy50kgyTz31lImLi7PqdejQwbRq1cravtD5v9j32s/Pz63NEydOmNDQUDN48ODzzt2FBKKznT592rhcLrNixQojyXz//ffWvjFjxhgfHx+zd+9eq2zBggVu4ezYsWMmNDTUdO3a1a3dwsJCc9NNN5nbb7/drT1J5vnnn7+gvvn5+ZmWLVuWWefpp582ksy3337rdoxznRuIjDHmhhtuKDHMFAWiW265xZw+fdoq37Fjh3E6neaRRx6xyi4kEBljzL59+4wkM2bMmGJ1a9SoYUaMGFHmOHFl4ZIZqjVjTIW298knn6hWrVrq2rWrTp06Zf3cfPPNioyMLPY0yo033qhGjRoVa6Nt27aKjo52a6NTp06SpBUrVrjVv/fee+Xl5eXWpiTt3LlTkvTTTz9p27ZtGjhwoPz8/Ers98mTJ/XFF1+oZ8+eCggIcDtu586ddfLkSesy3O23367vv/9eQ4YM0eeff67Dhw9f1BwNHz5ca9eu1dq1a7Vs2TKNHz9e7733nv74xz9adT7//HOdOnVKDz74oFtf/Pz81Lp16wt6qqdBgwaqX7++dYkiMzNT8fHx+tOf/qTt27dr27Ztys/P18qVK91urL/Q+b/Y9/rmm29WvXr1rG0/Pz81atTIep8u1S+//KJ+/fopMjJSXl5ecjqdat26taQzT3QV+fOf/yxJmjFjhlU2depUxcfH6+6775YkZWVl6cCBA+rfv7/b2E6fPq2kpCStXbvW7VKcdObevIpS9HtZ0mWyS9WvXz+3dmNjY9WqVSstW7asQo9z++23a/bs2Ro3bpxWr14tl8tVoe2j8nFTNaqtY8eOaf/+/YqPj6+wNvfu3atDhw7Jx8enxP2///6723ZUVFSJbXz88ccl3lNTUhu1a9d22/b19ZV05gZtSdb9LOfeLH62/fv369SpU0pLS1NaWlqZxx09erQCAwM1b948vfHGG/Ly8tLdd9+tl19+2e3m9dLUrVvXrV6bNm3kcDg0evRoff755+rYsaP27t0rSbrttttKbOOqqy7s/2rt2rVTRkaGpDP3w3To0EHx8fGKiIjQ0qVL1bBhQ504ccItEF3o/F/se33u+ySdea+K3qdLcfToUd11113y8/PTuHHj1KhRIwUEBOjXX39Vr1693I4RERGhvn37avr06frLX/6iTZs26euvv3a7l65o/u+7775Sj3ngwAG3e4BKOpdLUq9ePW3fvr3MOkX3BMXExFxQmxcjMjKyxLLvv/++Qo+zYMECjRs3Tm+++aaee+451ahRQz179tTEiRNL7AOqPgIRqq1PP/1UhYWFFfq4bFhYmGrXrm39I3yuoKAgt+2S/gccFhamG2+80e2m0rNFR0dfVJ+KbvzcvXt3qXVCQkLk5eWl5ORkDR06tMQ6cXFxkiRvb2+lpKQoJSVFhw4d0tKlS/XMM8+oY8eO+vXXX0t9uqgsRata33//vTp27KiwsDBJ0r/+9S/FxsZedHtF2rVrp5kzZ2rNmjX69ttv9b//+7+SpHvuuUeZmZnauXOnatSo4fbU24XO/8W+15fTl19+qT179mj58uXWqpB05mMNSjJ8+HDNnTtXH374oTIyMlSrVi23pyWL5j8tLc1tbs5W9GRmkQtdzenQoYP+8Y9/aPXq1SW2ffz4cWVmZqpp06ZWcCha2czPz7cCv1Q8dF6InJycEsvODqx+fn7Ky8srVu9ijhcWFqYpU6ZoypQp2rVrlz766CP95S9/UW5ubqnnDKo2AhGqpV27dmnUqFEKDg7W4MGDK6zdLl26aP78+SosLFSLFi3K3cbixYvVoEEDhYSEXHKfGjVqpAYNGmjWrFlKSUlx+welSEBAgNq2bav//Oc/uvHGG0td9ThXrVq1dN999+m3337TiBEjtGPHDl1//fUX3ccNGzZIksLDwyVJHTt2lLe3t7Zt23beSzFnr4id+5EF7dq1k8Ph0HPPPaerrrrKuiTUvn17/c///I927typu+++22016ELnvyLe64pSFEbOfW/PXvU5W/PmzdWqVSu9/PLL2rhxox599FG31Z477rhDtWrV0n//+98K/0DIJ598UrNmzdKwYcOKPXYvSaNGjdLBgwc1bdo0q6zoya4ffvjBbdXw7CcDi5xv1e3dd99VSkqKNWc7d+5UVlaWHnzwQbfjvf/++24BbP/+/crKylLNmjXdjiXpvKt89erV0+OPP64vvvhC33zzTZl1UXURiHDF27hxo3UPRG5urr7++mulp6fLy8tLCxcutFZQKsL999+vt99+W507d9bw4cN1++23y+l0avfu3Vq2bJm6d++unj17ltnGCy+8oMzMTLVq1UpPPPGEGjdurJMnT2rHjh1avHix3njjjTIvf5XkH//4h7p27aqWLVvqySefVL169bRr1y59/vnnevvttyVJf/vb33TnnXfqrrvu0p///GfVr19fR44c0c8//6yPP/5YX375paQzj64XfbZTnTp1tHPnTk2ZMkWxsbFq2LDhefuya9cu636kY8eOadWqVZowYYJiY2PVq1cvSWf+QXrhhRf07LPP6pdfflFSUpJCQkK0d+9erVmzRoGBgdZHFBRd8nz55ZfVqVMneXl5WaEuPDxcTZs21ZIlS9S2bVtr9ap9+/Y6cOCADhw4oEmTJpVr/ivivb4Y27Zt07/+9a9i5ddff71atWqlkJAQPfbYYxozZoycTqfefvvtMi8DDR8+XH379pXD4dCQIUPc9tWoUUNpaWnq37+/Dhw4oPvuu0/h4eHat2+fvv/+e+3bt88tsFyMBg0aaO7cuXrggQd02223KSUlxfpgxlmzZumzzz7TqFGj1LdvX+s1nTt3VmhoqAYOHKgXXnhB3t7emj17tn799ddi7cfHx2v+/PlasGCBrrnmGvn5+bldFs/NzVXPnj01aNAg5eXlacyYMfLz89Po0aOtOsnJyZo+fbr+9Kc/adCgQdq/f78mTpzoFoakM6uAsbGx+vDDD9WuXTuFhoYqLCxMISEhatu2rfr166frrrtOQUFBWrt2rTIyMqxzHFcgT9/VDZRX0RMoRT8+Pj4mPDzctG7d2owfP97k5uYWe82lPmVmzJlHmF999VVz0003GT8/P1OjRg1z3XXXmcGDB5utW7da9WJjY829995bYt/37dtnnnjiCRMXF2ecTqcJDQ01zZs3N88++6w5evSoMeb/PbX1yiuvFHu9SnjyZdWqVaZTp04mODjY+Pr6mgYNGpgnn3zSrc727dvNww8/bK6++mrjdDpNnTp1TKtWrcy4ceOsOq+99ppp1aqVCQsLMz4+PqZevXpm4MCBZseOHSWO5ey2dc7TZX5+fqZRo0ZmxIgR1iPWZ1u0aJFp27atqVmzpvH19TWxsbHmvvvuc3vsOz8/3zzyyCOmTp06xuFwFHvq6MknnzSSzEsvveTWdsOGDY0k88MPPxQ77oXMvzGX/l6X9jTTuc6dt7N/it7nrKwsk5CQYAICAkydOnXMI488Yr777jsjyaSnpxdrMz8/3/j6+pqkpKRSj7tixQpz7733mtDQUON0Os3VV19t7r33XvP+++9bdYp+Z/bt23fecZxt06ZNpn///qZu3brWHCclJZlPP/20xPpr1qwxrVq1MoGBgebqq682Y8aMMW+++Wax93vHjh0mMTHRBAUFWR8HYMz/e8ps7ty55oknnjB16tQxvr6+5q677jLr1q0rdrw5c+aYJk2aGD8/P3P99debBQsWFHvKzBhjli5dapo1a2Z8fX2NJNO/f39z8uRJ89hjj5kbb7zR1KxZ0/j7+5vGjRubMWPGmGPHjl3UPKHqcBhTwY/hAAA87uOPP1a3bt306aefqnPnzp7uzmW3fPlytW3bVu+//36ZN4sDpeGSGQBUI//973+1c+dOjRw5UjfffLP1cQIAysbnEAFANTJkyBB169ZNISEhevfddy/LZ/0A1RGXzAAAgO2xQgQAAGyPQAQAAGyPQAQAAGyPp8wu0OnTp7Vnzx4FBQVxkyIAAFcIY4yOHDmi6OjoMr8nkUB0gfbs2XNZvogQAABcfr/++muZ3wJAILpARV/k+Ouvvxb7ePfqxuVyacmSJUpMTCz1G8FR8Zh3z2HuPYN59xw7zf3hw4cVExNz3i9kJhBdoKLLZDVr1rRFIAoICFDNmjWr/S9KVcK8ew5z7xnMu+fYce7Pd7sLN1UDAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADbIxABAADb8/Z0BwCgqujSRSooKH3/smWV1xcAlYsVIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHseDUTTpk3TjTfeqJo1a6pmzZpKSEjQZ599Zu03xig1NVXR0dHy9/dXmzZttGnTJrc28vPzNWzYMIWFhSkwMFDdunXT7t273eocPHhQycnJCg4OVnBwsJKTk3Xo0KHKGCIAALgCeDQQ1a1bV3/961+1bt06rVu3Tvfcc4+6d+9uhZ6JEydq0qRJmjp1qtauXavIyEh16NBBR44csdoYMWKEFi5cqPnz52vlypU6evSounTposLCQqtOv379tGHDBmVkZCgjI0MbNmxQcnJypY8XAABUTd6ePHjXrl3dtl966SVNmzZNq1ev1vXXX68pU6bo2WefVa9evSRJc+bMUUREhN555x0NHjxYeXl5mjlzpubOnav27dtLkubNm6eYmBgtXbpUHTt21ObNm5WRkaHVq1erRYsWkqQZM2YoISFBW7ZsUePGjSt30AAAoMrxaCA6W2Fhod5//30dO3ZMCQkJ2r59u3JycpSYmGjV8fX1VevWrZWVlaXBgwdr/fr1crlcbnWio6PVtGlTZWVlqWPHjlq1apWCg4OtMCRJLVu2VHBwsLKyskoNRPn5+crPz7e2Dx8+LElyuVxyuVwVPfwqpWh81X2cVQ3z7jlFc+50lj33vDUVi3Pec+w09xc6Ro8Hoh9//FEJCQk6efKkatSooYULF+r6669XVlaWJCkiIsKtfkREhHbu3ClJysnJkY+Pj0JCQorVycnJseqEh4cXO254eLhVpyQTJkzQ2LFji5UvWbJEAQEBFzfIK1RmZqanu2BLzLvnDBpU9twvXlxJHbEZznnPscPcHz9+/ILqeTwQNW7cWBs2bNChQ4f073//W/3799eKFSus/Q6Hw62+MaZY2bnOrVNS/fO1M3r0aKWkpFjbhw8fVkxMjBITE1WzZs3zjutK5nK5lJmZqQ4dOsjpdHq6O7bBvHtO0dzPmNFBLlfpc//JJ5XYKRvgnPccO8190RWe8/F4IPLx8dG1114rSbr11lu1du1a/e1vf9PTTz8t6cwKT1RUlFU/NzfXWjWKjIxUQUGBDh486LZKlJubq1atWll19u7dW+y4+/btK7b6dDZfX1/5+voWK3c6ndX+5Clip7FWJcy757hcThUUlD73vC2XB+e859hh7i90fFXuc4iMMcrPz1dcXJwiIyPdlvMKCgq0YsUKK+w0b95cTqfTrU52drY2btxo1UlISFBeXp7WrFlj1fn222+Vl5dn1QEAAPbm0RWiZ555Rp06dVJMTIyOHDmi+fPna/ny5crIyJDD4dCIESM0fvx4NWzYUA0bNtT48eMVEBCgfv36SZKCg4M1cOBAjRw5UrVr11ZoaKhGjRql+Ph466mzJk2aKCkpSYMGDdL06dMlSY8++qi6dOnCE2YAAECShwPR3r17lZycrOzsbAUHB+vGG29URkaGOnToIEl66qmndOLECQ0ZMkQHDx5UixYttGTJEgUFBVltTJ48Wd7e3urTp49OnDihdu3aafbs2fLy8rLqvP3223riiSesp9G6deumqVOnVu5gAQBAleXRQDRz5swy9zscDqWmpio1NbXUOn5+fkpLS1NaWlqpdUJDQzVv3rzydhMAAFRzVe4eIgAAgMpGIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALbn0UA0YcIE3XbbbQoKClJ4eLh69OihLVu2uNUZMGCAHA6H20/Lli3d6uTn52vYsGEKCwtTYGCgunXrpt27d7vVOXjwoJKTkxUcHKzg4GAlJyfr0KFDl3uIAADgCuDRQLRixQoNHTpUq1evVmZmpk6dOqXExEQdO3bMrV5SUpKys7Otn8WLF7vtHzFihBYuXKj58+dr5cqVOnr0qLp06aLCwkKrTr9+/bRhwwZlZGQoIyNDGzZsUHJycqWMEwAAVG3enjx4RkaG23Z6errCw8O1fv163X333Va5r6+vIiMjS2wjLy9PM2fO1Ny5c9W+fXtJ0rx58xQTE6OlS5eqY8eO2rx5szIyMrR69Wq1aNFCkjRjxgwlJCRoy5Ytaty48WUaIQAAuBJ4NBCdKy8vT5IUGhrqVr58+XKFh4erVq1aat26tV566SWFh4dLktavXy+Xy6XExESrfnR0tJo2baqsrCx17NhRq1atUnBwsBWGJKlly5YKDg5WVlZWiYEoPz9f+fn51vbhw4clSS6XSy6Xq+IGXQUVja+6j7OqYd49p2jOnc6y5563pmJxznuOneb+QsdYZQKRMUYpKSm688471bRpU6u8U6dO6t27t2JjY7V9+3Y999xzuueee7R+/Xr5+voqJydHPj4+CgkJcWsvIiJCOTk5kqScnBwrQJ0tPDzcqnOuCRMmaOzYscXKlyxZooCAgEsZ6hUjMzPT012wJebdcwYNKnvuz7lajwrCOe85dpj748ePX1C9KhOIHn/8cf3www9auXKlW3nfvn2tPzdt2lS33nqrYmNj9emnn6pXr16ltmeMkcPhsLbP/nNpdc42evRopaSkWNuHDx9WTEyMEhMTVbNmzQse15XI5XIpMzNTHTp0kNPp9HR3bIN595yiuZ8xo4NcrtLn/pNPKrFTNsA57zl2mvuiKzznUyUC0bBhw/TRRx/pq6++Ut26dcusGxUVpdjYWG3dulWSFBkZqYKCAh08eNBtlSg3N1etWrWy6uzdu7dYW/v27VNERESJx/H19ZWvr2+xcqfTWe1PniJ2GmtVwrx7jsvlVEFB6XPP23J5cM57jh3m/kLH59GnzIwxevzxx/XBBx/oyy+/VFxc3Hlfs3//fv3666+KioqSJDVv3lxOp9Nt2S87O1sbN260AlFCQoLy8vK0Zs0aq863336rvLw8qw4AALAvj64QDR06VO+8844+/PBDBQUFWffzBAcHy9/fX0ePHlVqaqr+8Ic/KCoqSjt27NAzzzyjsLAw9ezZ06o7cOBAjRw5UrVr11ZoaKhGjRql+Ph466mzJk2aKCkpSYMGDdL06dMlSY8++qi6dOnCE2YAAMCzgWjatGmSpDZt2riVp6ena8CAAfLy8tKPP/6ot956S4cOHVJUVJTatm2rBQsWKCgoyKo/efJkeXt7q0+fPjpx4oTatWun2bNny8vLy6rz9ttv64knnrCeRuvWrZumTp16+QcJAACqPI8GImNMmfv9/f31+eefn7cdPz8/paWlKS0trdQ6oaGhmjdv3kX3EQAAVH98lxkAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9AhEAALA9jwaiCRMm6LbbblNQUJDCw8PVo0cPbdmyxa2OMUapqamKjo6Wv7+/2rRpo02bNrnVyc/P17BhwxQWFqbAwEB169ZNu3fvdqtz8OBBJScnKzg4WMHBwUpOTtahQ4cu9xABAMAVwKOBaMWKFRo6dKhWr16tzMxMnTp1SomJiTp27JhVZ+LEiZo0aZKmTp2qtWvXKjIyUh06dNCRI0esOiNGjNDChQs1f/58rVy5UkePHlWXLl1UWFho1enXr582bNigjIwMZWRkaMOGDUpOTq7U8QIAgKrJuzwv2r59u+Li4i754BkZGW7b6enpCg8P1/r163X33XfLGKMpU6bo2WefVa9evSRJc+bMUUREhN555x0NHjxYeXl5mjlzpubOnav27dtLkubNm6eYmBgtXbpUHTt21ObNm5WRkaHVq1erRYsWkqQZM2YoISFBW7ZsUePGjS95LAAA4MpVrhWia6+9Vm3bttW8efN08uTJCutMXl6eJCk0NFTSmeCVk5OjxMREq46vr69at26trKwsSdL69evlcrnc6kRHR6tp06ZWnVWrVik4ONgKQ5LUsmVLBQcHW3UAAIB9lWuF6Pvvv9esWbM0cuRIPf744+rbt68GDhyo22+/vdwdMcYoJSVFd955p5o2bSpJysnJkSRFRES41Y2IiNDOnTutOj4+PgoJCSlWp+j1OTk5Cg8PL3bM8PBwq8658vPzlZ+fb20fPnxYkuRyueRyucozxCtG0fiq+zirGubdc4rm3Okse+55ayoW57zn2GnuL3SM5QpETZs21aRJkzRx4kR9/PHHmj17tu688041bNhQAwcOVHJysurUqXNRbT7++OP64YcftHLlymL7HA6H27YxpljZuc6tU1L9stqZMGGCxo4dW6x8yZIlCggIKPPY1UVmZqanu2BLzLvnDBpU9twvXlxJHbEZznnPscPcHz9+/ILqlSsQWS/29lbPnj3VuXNnvf766xo9erRGjRql0aNHq2/fvnr55ZcVFRV13naGDRumjz76SF999ZXq1q1rlUdGRko6s8Jzdju5ubnWqlFkZKQKCgp08OBBt1Wi3NxctWrVyqqzd+/eYsfdt29fsdWnIqNHj1ZKSoq1ffjwYcXExCgxMVE1a9Y875iuZC6XS5mZmerQoYOcTqenu2MbzLvnFM39jBkd5HKVPveffFKJnbIBznnPsdPcF13hOZ9LCkTr1q3TrFmzNH/+fAUGBmrUqFEaOHCg9uzZo+eff17du3fXmjVrSn29MUbDhg3TwoULtXz58mI3asfFxSkyMlKZmZlq1qyZJKmgoEArVqzQyy+/LElq3ry5nE6nMjMz1adPH0lSdna2Nm7cqIkTJ0qSEhISlJeXpzVr1liX9b799lvl5eVZoelcvr6+8vX1LVbudDqr/clTxE5jrUqYd89xuZwqKCh97nlbLg/Oec+xw9xf6PjKFYgmTZqk9PR0bdmyRZ07d9Zbb72lzp0766qrztyjHRcXp+nTp+u6664rs52hQ4fqnXfe0YcffqigoCDrfp7g4GD5+/vL4XBoxIgRGj9+vBo2bKiGDRtq/PjxCggIUL9+/ay6AwcO1MiRI1W7dm2FhoZq1KhRio+Pt546a9KkiZKSkjRo0CBNnz5dkvToo4+qS5cuPGEGAADKF4imTZumhx9+WA899JB1Wetc9erV08yZM8/bjiS1adPGrTw9PV0DBgyQJD311FM6ceKEhgwZooMHD6pFixZasmSJgoKCrPqTJ0+Wt7e3+vTpoxMnTqhdu3aaPXu2vLy8rDpvv/22nnjiCetptG7dumnq1KkXO3QAAFANlSsQbd269bx1fHx81L9//zLrGGPO247D4VBqaqpSU1NLrePn56e0tDSlpaWVWic0NFTz5s077/EAAID9lOtziNLT0/X+++8XK3///fc1Z86cS+4UAABAZSpXIPrrX/+qsLCwYuXh4eEaP378JXcKAACgMpUrEO3cubPEr+6IjY3Vrl27LrlTAAAAlalcgSg8PFw//PBDsfLvv/9etWvXvuROAQAAVKZyBaL7779fTzzxhJYtW6bCwkIVFhbqyy+/1PDhw3X//fdXdB8BAAAuq3I9ZTZu3Djt3LlT7dq1k7f3mSZOnz6tBx98kHuIAADAFadcgcjHx0cLFizQiy++qO+//17+/v6Kj49XbGxsRfcPAADgsrukr+5o1KiRGjVqVFF9AQAA8IhyBaLCwkLNnj1bX3zxhXJzc3X69Gm3/V9++WWFdA4AAKAylCsQDR8+XLNnz9a9996rpk2byuFwVHS/AAAAKk25AtH8+fP13nvvqXPnzhXdHwAAgEpXrsfufXx8dO2111Z0XwAAADyiXIFo5MiR+tvf/nZBX84KAABQ1ZXrktnKlSu1bNkyffbZZ7rhhhvkdDrd9n/wwQcV0jkAAIDKUK5AVKtWLfXs2bOi+wIAAOAR5QpE6enpFd0PAAAAjynXPUSSdOrUKS1dulTTp0/XkSNHJEl79uzR0aNHK6xzAAAAlaFcK0Q7d+5UUlKSdu3apfz8fHXo0EFBQUGaOHGiTp48qTfeeKOi+wkAAHDZlGuFaPjw4br11lt18OBB+fv7W+U9e/bUF198UWGdAwAAqAzlfsrsm2++kY+Pj1t5bGysfvvttwrpGAAAQGUp1wrR6dOnVVhYWKx89+7dCgoKuuROAQAAVKZyBaIOHTpoypQp1rbD4dDRo0c1ZswYvs4DAABcccp1yWzy5Mlq27atrr/+ep08eVL9+vXT1q1bFRYWpnfffbei+wgAAHBZlSsQRUdHa8OGDXr33Xf13Xff6fTp0xo4cKAeeOABt5usAQAArgTlCkSS5O/vr4cfflgPP/xwRfYHAACg0pUrEL311ltl7n/wwQfL1RkAAABPKFcgGj58uNu2y+XS8ePH5ePjo4CAAAIRAAC4opTrKbODBw+6/Rw9elRbtmzRnXfeyU3VAADgilPu7zI7V8OGDfXXv/612OoRAABAVVdhgUiSvLy8tGfPnopsEgAA4LIr1z1EH330kdu2MUbZ2dmaOnWq7rjjjgrpGAAAQGUpVyDq0aOH27bD4VCdOnV0zz336LXXXquIfgEAAFSacgWi06dPV3Q/AAAAPKZC7yECAAC4EpVrhSglJeWC606aNKk8hwAAAKg05QpE//nPf/Tdd9/p1KlTaty4sSTpp59+kpeXl2655RarnsPhqJheAgAAXEblCkRdu3ZVUFCQ5syZo5CQEElnPqzxoYce0l133aWRI0dWaCcBAAAup3LdQ/Taa69pwoQJVhiSpJCQEI0bN46nzAAAwBWnXIHo8OHD2rt3b7Hy3NxcHTly5JI7BQAAUJnKFYh69uyphx56SP/617+0e/du7d69W//61780cOBA9erVq6L7CAAAcFmV6x6iN954Q6NGjdKf/vQnuVyuMw15e2vgwIF65ZVXKrSDAAAAl1u5AlFAQIBef/11vfLKK9q2bZuMMbr22msVGBhY0f0DAAC47C7pgxmzs7OVnZ2tRo0aKTAwUMaYiuoXAABApSlXINq/f7/atWunRo0aqXPnzsrOzpYkPfLIIxf1yP1XX32lrl27Kjo6Wg6HQ4sWLXLbP2DAADkcDrefli1butXJz8/XsGHDFBYWpsDAQHXr1k27d+92q3Pw4EElJycrODhYwcHBSk5O1qFDh8ozdAAAUA2VKxA9+eSTcjqd2rVrlwICAqzyvn37KiMj44LbOXbsmG666SZNnTq11DpJSUnWSlR2drYWL17stn/EiBFauHCh5s+fr5UrV+ro0aPq0qWLCgsLrTr9+vXThg0blJGRoYyMDG3YsEHJyckXMWIAAFCdleseoiVLlujzzz9X3bp13cobNmyonTt3XnA7nTp1UqdOncqs4+vrq8jIyBL35eXlaebMmZo7d67at28vSZo3b55iYmK0dOlSdezYUZs3b1ZGRoZWr16tFi1aSJJmzJihhIQEbdmyxfqkbQAAYF/lCkTHjh1zWxkq8vvvv8vX1/eSO3W25cuXKzw8XLVq1VLr1q310ksvKTw8XJK0fv16uVwuJSYmWvWjo6PVtGlTZWVlqWPHjlq1apWCg4OtMCRJLVu2VHBwsLKyskoNRPn5+crPz7e2Dx8+LElyuVzWk3XVVdH4qvs4qxrm3XOK5tzpLHvueWsqFue859hp7i90jOUKRHfffbfeeustvfjii5LOfGfZ6dOn9corr6ht27blabJEnTp1Uu/evRUbG6vt27frueee0z333KP169fL19dXOTk58vHxcfvEbEmKiIhQTk6OJCknJ8cKUGcLDw+36pRkwoQJGjt2bLHyJUuWlBgGq6PMzExPd8GWmHfPGTSo7Lk/54o9KgjnvOfYYe6PHz9+QfXKFYheeeUVtWnTRuvWrVNBQYGeeuopbdq0SQcOHNA333xTniZL1LdvX+vPTZs21a233qrY2Fh9+umnZX4ApDHG7YtlS/qS2XPrnGv06NFKSUmxtg8fPqyYmBglJiaqZs2aFzuUK4rL5VJmZqY6dOggp9Pp6e7YBvPuOUVzP2NGB7lcpc/9J59UYqdsgHPec+w090VXeM6nXIHo+uuv1w8//KBp06bJy8tLx44dU69evTR06FBFRUWVp8kLEhUVpdjYWG3dulWSFBkZqYKCAh08eNBtlSg3N1etWrWy6pT0NSP79u1TREREqcfy9fUt8fKf0+ms9idPETuNtSph3j3H5XKqoKD0uedtuTw45z3HDnN/oeO76EBUdM/O9OnTS7ykdDnt379fv/76qxW6mjdvLqfTqczMTPXp00fSmc9G2rhxoyZOnChJSkhIUF5entasWaPbb79dkvTtt98qLy/PCk0AAMDeLjoQOZ1Obdy4sczLTRfq6NGj+vnnn63t7du3a8OGDQoNDVVoaKhSU1P1hz/8QVFRUdqxY4eeeeYZhYWFqWfPnpKk4OBgDRw4UCNHjlTt2rUVGhqqUaNGKT4+3nrqrEmTJkpKStKgQYM0ffp0SdKjjz6qLl268IQZAACQVM7PIXrwwQc1c+bMSz74unXr1KxZMzVr1kySlJKSombNmun555+Xl5eXfvzxR3Xv3l2NGjVS//791ahRI61atUpBQUFWG5MnT1aPHj3Up08f3XHHHQoICNDHH38sLy8vq87bb7+t+Ph4JSYmKjExUTfeeKPmzp17yf0HAADVQ7nuISooKNCbb76pzMxM3XrrrcW+w2zSpEkX1E6bNm3K/LqPzz///Lxt+Pn5KS0tTWlpaaXWCQ0N1bx58y6oTwAAwH4uKhD98ssvql+/vjZu3KhbbrlFkvTTTz+51amIS2kAAACV6aICUcOGDZWdna1ly5ZJOvNY/N///vcyn9YCAACo6i7qHqJzL2999tlnOnbsWIV2CAAAoLKV66bqImXd/wMAAHCluKhA5HA4it0jxD1DAADgSndR9xAZYzRgwADrE5xPnjypxx57rNhTZh988EHF9RAAAOAyu6hA1L9/f7ftP/3pTxXaGQAAAE+4qECUnp5+ufoBAADgMZd0UzUAAEB1QCACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC259FA9NVXX6lr166Kjo6Ww+HQokWL3PYbY5Samqro6Gj5+/urTZs22rRpk1ud/Px8DRs2TGFhYQoMDFS3bt20e/dutzoHDx5UcnKygoODFRwcrOTkZB06dOgyjw4AAFwpPBqIjh07pptuuklTp04tcf/EiRM1adIkTZ06VWvXrlVkZKQ6dOigI0eOWHVGjBihhQsXav78+Vq5cqWOHj2qLl26qLCw0KrTr18/bdiwQRkZGcrIyNCGDRuUnJx82ccHAACuDN6ePHinTp3UqVOnEvcZYzRlyhQ9++yz6tWrlyRpzpw5ioiI0DvvvKPBgwcrLy9PM2fO1Ny5c9W+fXtJ0rx58xQTE6OlS5eqY8eO2rx5szIyMrR69Wq1aNFCkjRjxgwlJCRoy5Ytaty4ceUMFgAAVFkeDURl2b59u3JycpSYmGiV+fr6qnXr1srKytLgwYO1fv16uVwutzrR0dFq2rSpsrKy1LFjR61atUrBwcFWGJKkli1bKjg4WFlZWaUGovz8fOXn51vbhw8fliS5XC65XK6KHm6VUjS+6j7OqoZ595yiOXc6y5573pqKxTnvOXaa+wsdY5UNRDk5OZKkiIgIt/KIiAjt3LnTquPj46OQkJBidYpen5OTo/Dw8GLth4eHW3VKMmHCBI0dO7ZY+ZIlSxQQEHBxg7lCZWZmeroLtsS8e86gQWXP/eLFldQRm+Gc9xw7zP3x48cvqF6VDURFHA6H27YxpljZuc6tU1L987UzevRopaSkWNuHDx9WTEyMEhMTVbNmzQvt/hXJ5XIpMzNTHTp0kNPp9HR3bIN595yiuZ8xo4NcrtLn/pNPKrFTNsA57zl2mvuiKzznU2UDUWRkpKQzKzxRUVFWeW5urrVqFBkZqYKCAh08eNBtlSg3N1etWrWy6uzdu7dY+/v27Su2+nQ2X19f+fr6Fit3Op3V/uQpYqexViXMu+e4XE4VFJQ+97wtlwfnvOfYYe4vdHxV9nOI4uLiFBkZ6bacV1BQoBUrVlhhp3nz5nI6nW51srOztXHjRqtOQkKC8vLytGbNGqvOt99+q7y8PKsOAACwN4+uEB09elQ///yztb19+3Zt2LBBoaGhqlevnkaMGKHx48erYcOGatiwocaPH6+AgAD169dPkhQcHKyBAwdq5MiRql27tkJDQzVq1CjFx8dbT501adJESUlJGjRokKZPny5JevTRR9WlSxeeMAMAAJI8HIjWrVuntm3bWttF9+z0799fs2fP1lNPPaUTJ05oyJAhOnjwoFq0aKElS5YoKCjIes3kyZPl7e2tPn366MSJE2rXrp1mz54tLy8vq87bb7+tJ554wnoarVu3bqV+9hEAALAfjwaiNm3ayBhT6n6Hw6HU1FSlpqaWWsfPz09paWlKS0srtU5oaKjmzZt3KV0FAADVWJW9hwgAAKCyEIgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtVelAlJqaKofD4fYTGRlp7TfGKDU1VdHR0fL391ebNm20adMmtzby8/M1bNgwhYWFKTAwUN26ddPu3bsreygAAKAKq9KBSJJuuOEGZWdnWz8//vijtW/ixImaNGmSpk6dqrVr1yoyMlIdOnTQkSNHrDojRozQwoULNX/+fK1cuVJHjx5Vly5dVFhY6InhAACAKsjb0x04H29vb7dVoSLGGE2ZMkXPPvusevXqJUmaM2eOIiIi9M4772jw4MHKy8vTzJkzNXfuXLVv316SNG/ePMXExGjp0qXq2LFjpY4FAABUTVU+EG3dulXR0dHy9fVVixYtNH78eF1zzTXavn27cnJylJiYaNX19fVV69atlZWVpcGDB2v9+vVyuVxudaKjo9W0aVNlZWWVGYjy8/OVn59vbR8+fFiS5HK55HK5LsNIq46i8VX3cVY1zLvnFM2501n23PPWVCzOec+x09xf6BirdCBq0aKF3nrrLTVq1Eh79+7VuHHj1KpVK23atEk5OTmSpIiICLfXREREaOfOnZKknJwc+fj4KCQkpFidoteXZsKECRo7dmyx8iVLliggIOBShnXFyMzM9HQXbIl595xBg8qe+8WLK6kjNsM57zl2mPvjx49fUL0qHYg6depk/Tk+Pl4JCQlq0KCB5syZo5YtW0qSHA6H22uMMcXKznUhdUaPHq2UlBRr+/Dhw4qJiVFiYqJq1qx5sUO5orhcLmVmZqpDhw5yOp2e7o5tMO+eUzT3M2Z0kMtV+tx/8kkldsoGOOc9x05zX3SF53yqdCA6V2BgoOLj47V161b16NFD0plVoKioKKtObm6utWoUGRmpgoICHTx40G2VKDc3V61atSrzWL6+vvL19S1W7nQ6q/3JU8ROY61KmHfPcbmcKigofe55Wy4PznnPscPcX+j4qvxTZmfLz8/X5s2bFRUVpbi4OEVGRrot9xUUFGjFihVW2GnevLmcTqdbnezsbG3cuPG8gQgAANhHlV4hGjVqlLp27ap69eopNzdX48aN0+HDh9W/f385HA6NGDFC48ePV8OGDdWwYUONHz9eAQEB6tevnyQpODhYAwcO1MiRI1W7dm2FhoZq1KhRio+Pt546AwAAqNKBaPfu3frjH/+o33//XXXq1FHLli21evVqxcbGSpKeeuopnThxQkOGDNHBgwfVokULLVmyREFBQVYbkydPlre3t/r06aMTJ06oXbt2mj17try8vDw1LAAAUMVU6UA0f/78Mvc7HA6lpqYqNTW11Dp+fn5KS0tTWlpaBfcOAABUF1fUPUQAAACXA4EIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYnrenOwAAl1vbtmXv9/GRhgypnL4AqJoIRACuaOcLO5V9rGXLLn8/AFQ8LpkBAADbY4UIgEfYebXFzmMHqioCEQAL/1ADsCsumQEAANtjhQgAqjFW/YALwwoRAACwPVaIgHLgf90AUL3YaoXo9ddfV1xcnPz8/NS8eXN9/fXXnu4SAACoAmyzQrRgwQKNGDFCr7/+uu644w5Nnz5dnTp10n//+1/Vq1fP090DAFwEVmlR0WyzQjRp0iQNHDhQjzzyiJo0aaIpU6YoJiZG06ZN83TXAACAh9kiEBUUFGj9+vVKTEx0K09MTFRWVpaHegUAAKoKW1wy+/3331VYWKiIiAi38oiICOXk5JT4mvz8fOXn51vbeXl5kqQDBw7I5XJVaP/69Dl/nffeq9BDlsnlcun48ePav3+/nE5n5R34CnLVBfxXYv/+i2uzKsz75RjXpRyrXbuKaef8zsy9tF9XXXVpc19Rfa7Mea6oY12sSznnq/K4rgSV/feNJ/+dO3LkiCTJGFNmPVsEoiIOh8Nt2xhTrKzIhAkTNHbs2GLlcXFxl6Vv5xMW5pHD4hJU1/esuo5ryRJP98BdZc5zdX1Pq+u4qqvL/X4dOXJEwcHBpe63RSAKCwuTl5dXsdWg3NzcYqtGRUaPHq2UlBRr+/Tp0zpw4IBq165daoiqLg4fPqyYmBj9+uuvqlmzpqe7YxvMu+cw957BvHuOnebeGKMjR44oOjq6zHq2CEQ+Pj5q3ry5MjMz1bNnT6s8MzNT3bt3L/E1vr6+8vX1dSurVavW5exmlVOzZs1q/4tSFTHvnsPcewbz7jl2mfuyVoaK2CIQSVJKSoqSk5N16623KiEhQf/85z+1a9cuPfbYY57uGgAA8DDbBKK+fftq//79euGFF5Sdna2mTZtq8eLFio2N9XTXAACAh9kmEEnSkCFDNGTIEE93o8rz9fXVmDFjil0yxOXFvHsOc+8ZzLvnMPfFOcz5nkMDAACo5mzxwYwAAABlIRABAADbIxABAADbIxABAADbIxChVDt27NDAgQMVFxcnf39/NWjQQGPGjFFBQYGnu1btvfTSS2rVqpUCAgJs94Ggle31119XXFyc/Pz81Lx5c3399dee7lK199VXX6lr166Kjo6Ww+HQokWLPN0lW5gwYYJuu+02BQUFKTw8XD169NCWLVs83a0qg0CEUv3f//2fTp8+renTp2vTpk2aPHmy3njjDT3zzDOe7lq1V1BQoN69e+vPf/6zp7tSrS1YsEAjRozQs88+q//85z+666671KlTJ+3atcvTXavWjh07pptuuklTp071dFdsZcWKFRo6dKhWr16tzMxMnTp1SomJiTp27Jinu1Yl8Ng9Lsorr7yiadOm6ZdffvF0V2xh9uzZGjFihA4dOuTprlRLLVq00C233KJp06ZZZU2aNFGPHj00YcIED/bMPhwOhxYuXKgePXp4uiu2s2/fPoWHh2vFihW6++67Pd0dj2OFCBclLy9PoaGhnu4GcMkKCgq0fv16JSYmupUnJiYqKyvLQ70CKk9eXp4k8Xf6/49AhAu2bds2paWl8f1vqBZ+//13FRYWKiIiwq08IiJCOTk5HuoVUDmMMUpJSdGdd96ppk2bero7VQKByIZSU1PlcDjK/Fm3bp3ba/bs2aOkpCT17t1bjzzyiId6fmUrz7zj8nM4HG7bxphiZUB18/jjj+uHH37Qu+++6+muVBm2+i4znPH444/r/vvvL7NO/fr1rT/v2bNHbdu2VUJCgv75z39e5t5VXxc777i8wsLC5OXlVWw1KDc3t9iqEVCdDBs2TB999JG++uor1a1b19PdqTIIRDYUFhamsLCwC6r722+/qW3btmrevLnS09N11VUsKpbXxcw7Lj8fHx81b95cmZmZ6tmzp1WemZmp7t27e7BnwOVhjNGwYcO0cOFCLV++XHFxcZ7uUpVCIEKp9uzZozZt2qhevXp69dVXtW/fPmtfZGSkB3tW/e3atUsHDhzQrl27VFhYqA0bNkiSrr32WtWoUcOznatGUlJSlJycrFtvvdVaAd21axf3yV1mR48e1c8//2xtb9++XRs2bFBoaKjq1avnwZ5Vb0OHDtU777yjDz/8UEFBQdbqaHBwsPz9/T3cO8/jsXuUavbs2XrooYdK3Mdpc3kNGDBAc+bMKVa+bNkytWnTpvI7VI29/vrrmjhxorKzs9W0aVNNnjyZR5Avs+XLl6tt27bFyvv376/Zs2dXfodsorR749LT0zVgwIDK7UwVRCACAAC2xw0hAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAKo8h8OhRYsWWdv/93//p5YtW8rPz08333xzqWUAKt9XX32lrl27Kjo6utjv7uVQv379Er8se+jQoRfVDoEIgMcMGDDA+svL6XQqIiJCHTp00KxZs3T69GmrXnZ2tjp16mRtjxkzRoGBgdqyZYu++OKLUssAVL5jx47ppptu0tSpUyvleGvXrlV2drb1k5mZKUnq3bv3RbVDIALgUUlJScrOztaOHTv02WefqW3btho+fLi6dOmiU6dOSTrz3Xm+vr7Wa7Zt26Y777xTsbGxql27dqllF6ugoODSBwTYXKdOnTRu3Dj16tWrxP0FBQV66qmndPXVVyswMFAtWrTQ8uXLy328OnXqKDIy0vr55JNP1KBBA7Vu3fqi2iEQAfAoX19fRUZG6uqrr9Ytt9yiZ555Rh9++KE+++wz63utzl52dzgcWr9+vV544QU5HA6lpqaWWCZJv/32m/r27auQkBDVrl1b3bt3144dO6xjDxgwQD169NCECRMUHR2tRo0aXdTrXn31VUVFRal27doaOnSoXC6XVSc/P19PPfWUYmJi5Ovrq4YNG2rmzJnW/v/+97/q3LmzatSooYiICCUnJ+v333+/LHMMVCUPPfSQvvnmG82fP18//PCDevfuraSkJG3duvWS2y4oKNC8efP08MMPl/rdbaUhEAGocu655x7ddNNN+uCDD4rty87O1g033KCRI0cqOztbo0aNKrHs+PHjatu2rWrUqKGvvvpKK1euVI0aNZSUlOS2EvTFF19o8+bNyszM1CeffHLBr1u2bJm2bdumZcuWac6cOZo9e7bbF5M++OCDmj9/vv7+979r8+bNeuONN1SjRg1rDK1bt9bNN9+sdevWKSMjQ3v37lWfPn0u36QCVcC2bdv07rvv6v3339ddd92lBg0aaNSoUbrzzjuVnp5+ye0vWrRIhw4dKteX1Xpf8tEB4DK47rrr9MMPPxQrj4yMlLe3t2rUqKHIyEhJUo0aNYqVzZo1S1dddZXefPNN63+K6enpqlWrlpYvX67ExERJUmBgoN588035+Phc1OtCQkI0depUeXl56brrrtO9996rL774QoMGDdJPP/2k9957T5mZmWrfvr0k6ZprrrHGMG3aNN1yyy0aP368VTZr1izFxMTop59+slaqgOrmu+++kzGm2Dmen59vXeresWOH4uLiymxn6NChJd6jNHPmTHXq1EnR0dEX3TcCEYAqyRhz0UveZ1u/fr1+/vlnBQUFuZWfPHlS27Zts7bj4+OtMHQxr7vhhhvk5eVlbUdFRenHH3+UJG3YsEFeXl6l3sOwfv16LVu2zFoxOtu2bdsIRKi2Tp8+LS8vL61fv97t90eS9ftw9dVXa/PmzWW2ExISUqxs586dWrp0aYkryxeCQASgStq8efN5/5dYltOnT6t58+Z6++23i+2rU6eO9efAwMByvc7pdLrtczgc1pNx/v7+5+1b165d9fLLLxfbFxUVVeZrgStZs2bNVFhYqNzcXN11110l1nE6nbruuusuuu309HSFh4fr3nvvLVffCEQAqpwvv/xSP/74o5588slyt3HLLbdowYIFCg8PV82aNS/7684WHx+v06dPa8WKFdYls3OP8e9//1v169eXtzd/DaN6OXr0qH7++Wdre/v27dqwYYNCQ0PVqFEjPfDAA3rwwQf12muvqVmzZvr999/15ZdfKj4+Xp07dy7XMU+fPq309HT179+/3L9T3FQNwKPy8/OVk5Oj3377Td99953Gjx+v7t27q0uXLnrwwQfL3e4DDzygsLAwde/eXV9//bW2b9+uFStWaPjw4dq9e3eFv+5s9evXV//+/fXwww9r0aJF2r59u5YvX6733ntP0pn7Hw4cOKA//vGPWrNmjX755RctWbJEDz/8sAoLC8s9ZqAqWLdunZo1a6ZmzZpJklJSUtSsWTM9//zzks6s5Dz44IMaOXKkGjdurG7duunbb79VTExMuY+5dOlS7dq1Sw8//HC52+C/JgA8KiMjQ1FRUfL29lZISIhuuukm/f3vf1f//v111VXl/z9bQECAvvrqKz399NPq1auXjhw5oquvvlrt2rUrc+WnvK8717Rp0/TMM89oyJAh2r9/v+rVq6dnnnlGkhQdHa1vvvlGTz/9tDp27Kj8/HzFxsYqKSnpksYMVAVt2rSRMabU/U6nU2PHjtXYsWMr7JiJiYllHvNCOMyltgAAAHCF478iAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9v4/1PkCnvC0QMYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "differences = hf_model.model.layers[0](test_tensor)[0] - tl_model.blocks[0](test_tensor)\n",
    "\n",
    "# Flatten the differences to create a one-dimensional tensor\n",
    "flattened_differences = differences.flatten().cpu().detach().numpy()\n",
    "\n",
    "# Plot the histogram of the differences\n",
    "plt.hist(flattened_differences, bins=50, alpha=0.75, color='blue')\n",
    "plt.title('Differences Between Layer Outputs')\n",
    "plt.xlabel('Difference')\n",
    "plt.ylabel('Frequency')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e05fb0b6-b05d-4651-a976-2772a4177a0a",
   "metadata": {},
   "source": [
    "## Compare MLP Outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "44a55507-e639-414a-a297-e68e1c0696f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    tl_model.blocks[0].mlp.experts[0].W_in ==\n",
    "    hf_model.model.layers[0].block_sparse_moe.experts[0].w3.weight.T\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "03944deb-aa8d-46ff-83dd-4f7ee955656c",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_tensor = torch.randn((1, 1, 4096,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "eb0109ee-b82a-4ea0-b50b-8e6408647cea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(False)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.all(\n",
    "    hf_model.model.layers[0].block_sparse_moe(test_tensor)[0] ==\n",
    "    tl_model.blocks[0].mlp(test_tensor)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "25ce75bf-706e-4ae8-8f74-bc9c40e88c25",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.4624, -0.3203,  0.3846,  ...,  0.5780,  0.2270,  0.3475]]],\n",
       "       grad_fn=<ReshapeAliasBackward0>)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].block_sparse_moe(test_tensor)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "c016430e-0a30-426b-bfd0-0b1b423b3ff6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.4624, -0.3203,  0.3846,  ...,  0.5780,  0.2270,  0.3475]]],\n",
       "       grad_fn=<IndexPutBackward0>)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].mlp(test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "46353486-0a3f-4241-9cf5-ed25c7539f71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 1, 4096])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].mlp(test_tensor).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e25ada54-4e3c-42b7-8f35-ba67bfa500e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[False, False, False,  ..., False, False, False]]])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].block_sparse_moe(test_tensor)[0] == tl_model.blocks[0].mlp(test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "8f3a2865-645d-4441-95fb-32446f866760",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(201)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.sum(hf_model.model.layers[0].block_sparse_moe(test_tensor)[0] == tl_model.blocks[0].mlp(test_tensor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c6ef1f5e-bdf0-45e5-9347-6972e91e2f2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAHFCAYAAAAUpjivAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABMDklEQVR4nO3deViU5f4/8PcIw7AICCIMJCKHQFTIFEvFElHAJVxPobkbeenRTFy+pi1HLA+mpVmamoWgGWGpWLmjomZQKmrmkluISyDHDXAbRrh/f/jjOY5swzAww+P7dV1cNc/ccz+fzzwz+OZZZhRCCAEiIiIimWpg6gKIiIiIahPDDhEREckaww4RERHJGsMOERERyRrDDhEREckaww4RERHJGsMOERERyRrDDhEREckaww4RERHJGsMO1TuJiYlQKBTSj7W1NdRqNUJDQzF37lzk5eWVeUxsbCwUCoXOsqKiIowbNw7u7u6wsLDAs88+CwC4ceMGBg8eDFdXVygUCvTv378Ouqq/Lly4oLM9FAoFHBwc0KZNGyxatAjFxcUGzbtlyxbExsYat1gzU/p8jRo1qtz733//fWnMhQsXpOWjRo1Cw4YNK5378feJpaUlmjZtitGjR+PKlSt61Xf9+nXMnDkTrVq1gq2tLRwcHNCxY0d8/vnn0Gq1+rZZRl1v26VLlyIxMbHO1kdmSBDVMwkJCQKASEhIEBkZGWLfvn1i3bp1IiYmRjg6OgpnZ2eRmpqq85hLly6JjIwMnWWLFi0SAMTixYtFenq6OHbsmBBCiJiYGGFlZSXWrFkjMjIyxOnTp+ust/ooKytLABATJ04UGRkZIiMjQ2zdulX861//EgDElClTDJp3woQJQu6/ogAIe3t7YWtrKwoKCnTuKykpEd7e3sLBwUEAEFlZWdJ9I0eOFHZ2dpXO/fj7ZPfu3SI2NlaoVCrh7e0tbt++XenjT506JTw9PYWTk5OYM2eO2L17t9i8ebP417/+JSwsLERISIi4c+eOQX3X9bZt3bq1CAkJqbP1kfmR928SkqXSX+IHDx4sc192drbw9PQU9vb2Ijc3t9J5Xn/9dWFjY1NmeVhYmGjZsqXR6hVCiLt37xp1PnNSGnY++uijMve9+OKLwt3d3aB5n5SwM2zYMGFjYyNWrFihc9/OnTsFADFmzJgahZ3H3yfvvfeeACDWrFlT4WMfPHggWrVqJRwdHcsN+8nJyQKAGDt2rB5dlsWwQ3WNh7FIVpo1a4YFCxagsLAQX3zxhbT88cNYCoUCX331Fe7duyft5i/d7b9z506cOnVKWr5nzx4ADw97zZkzB/7+/lCpVGjSpAlGjx6N//73vzo1NG/eHJGRkdiwYQPatm0La2trzJ49GwCQm5uLsWPHomnTprCysoK3tzdmz56NBw8eSI8vPSz08ccfY+HChfD29kbDhg3RqVMn/Prrr2V6/u2339CnTx80btwY1tbW8PHxQUxMjM6Ys2fPYsiQIXB1dYVKpULLli3x+eef64wpKSnBnDlz0KJFC9jY2KBRo0Z45pln8Omnnxq0LQDA0dERSqWyzPK1a9eiU6dOsLOzQ8OGDdGjRw8cOXJEun/UqFFSfY8eirlw4QJeeeUVtG7dWme+Pn36QKFQ4Pvvv5eWHT58GAqFAj/99JO0TJ/nH6j+tt62bRvatWsHGxsb+Pv7Y+XKldV6jgYMGFDmMStXrkTnzp3h5+en91z66NixIwAgOzu7wjEpKSk4efIkZsyYUe76Bw0ahIiICMTHxyM3NxcAsGfPHp33S6nS13PpYaTKtm3psjfeeANffPEF/Pz8oFKp0KpVKyQnJ+vMW96haeB/h+9K52vevDlOnDiBvXv3Sutq3rw5gNp5zZN5sjR1AUTG1rt3b1hYWGDfvn0VjsnIyMAHH3yAtLQ07N69GwDg7e2NjIwMjB8/Hvn5+fjmm28AAK1atUJJSQn69euHn3/+GdOnT0dwcDCys7Mxa9YsdO3aFYcOHYKNjY00/+HDh3Hq1Cm8++678Pb2hp2dHXJzc/H888+jQYMG+Pe//w0fHx9kZGRgzpw5uHDhAhISEnRq/Pzzz+Hv749FixYBAN577z307t0bWVlZcHR0BABs374dffr0QcuWLbFw4UI0a9YMFy5cwI4dO6R5Tp48ieDgYCkIqtVqbN++HW+++SauXbuGWbNmAQDmz5+P2NhYvPvuu+jSpQu0Wi3+/PNP3Lp1S6/nvaSkRAoN+fn5+OGHH7Bt2za89dZbOuPi4uLw7rvvYvTo0Xj33XdRVFSEjz76CC+++CIOHDiAVq1a4b333sOdO3ewbt06ZGRkSI91d3dHWFgY1q1bh5ycHLi7u+PBgwfYu3cvbGxskJqaildeeQUAsHPnTlhaWqJr164AoPfzX91t/fvvv2Pq1KmYMWMG3Nzc8NVXXyE6OhpPP/00unTpotdzFx0dje7du+PUqVNo2bIlbt26hQ0bNmDp0qW4fv26XnPo69y5cwCAJk2aVDgmNTUVACo9X61///7YsWMH9uzZg8GDB+u9/sq2bakff/wRaWlpeP/992FnZ4elS5fi1VdfhaWlJV5++WW91wU8DG4vv/wyHB0dsXTpUgCASqUCUPPXPNUjpt61RFRdlR3GKuXm5qZzKGrWrFlldptXdCggJCREtG7dWmfZt99+KwCI9evX6yw/ePCgACCWLl0qLfPy8hIWFhZldv+PHTtWNGzYUGRnZ+ss//jjjwUAceLECSHE/w4LBQYGigcPHkjjDhw4IACIb7/9Vlrm4+MjfHx8xL179yp8Lnr06CGaNm0q8vPzdZa/8cYbwtraWty4cUMIIURkZKR49tlnK5ynIqX1lvczatQonR4uXrwoLC0txcSJE3XmKCwsFGq1WkRFRUnLKjrUce7cOQFArF69WgghxP79+wUAMX36dOHt7S2NCw8PF8HBwdJtfZ//6m5ra2trnTnv3bsnnJ2d9TrEA0BMmDBBOj9n2rRpQgghPv/8c9GwYUNRWFgoPvrooxodxvr111+FVqsVhYWFYtOmTaJJkyZVHubt2bOnACDu379f4ZitW7cKAGLevHlCCCHS0tIEAJGWlqYzrvT1kZCQIC2r7DAWAGFjY6NT34MHD4S/v794+umnpWXlvacf7fvR56uiw1iGvuap/uFhLJIlIYRR59u0aRMaNWqEPn364MGDB9LPs88+C7VaXWbX/TPPPFNm9/+mTZsQGhoKDw8PnTl69eoFANi7d6/O+JdeegkWFhY6cwL/O/xw5swZnD9/HtHR0bC2ti637vv372PXrl0YMGAAbG1tddbbu3dv3L9/Xzo09vzzz+P333/H+PHjsX37dhQUFFTrOZo0aRIOHjyIgwcPIi0tDXFxcfjuu+/w6quvSmO2b9+OBw8eYMSIETq1WFtbIyQkpMzzWB4fHx80b94cO3fuBPBwL0RgYCCGDRuGrKwsnD9/HhqNBvv370dYWJj0OH2f/+pu62effRbNmjWTbltbW8PPz6/Sw0SPK70i6+uvv8aDBw8QHx+PqKioKq+40kfHjh2hVCphb2+PyMhIqNVqbN26FW5ubjWat/Q9Vt6hpJrq3r27Tn0WFhYYNGgQzp07h8uXLxttPTV9zVP9wcNYJDt37tzB9evXERgYaLQ5r169ilu3bsHKyqrc+69du6Zz+9Fd8o/O8dNPP5V7Dkt5czRu3Fjndumu93v37gGAdP5I06ZNK6z7+vXrePDgARYvXozFixdXut6ZM2fCzs4Oa9aswfLly2FhYYEuXbpg3rx5aN++fYXrKNW0aVOdcV27doVCocDMmTOxfft29OjRA1evXgUAPPfcc+XO0aCBfn9/de/eHdu2bQPw8HBVeHg4AgMD4ebmhp07d8LX1xf37t3TCTv6Pv/V3daPbyfg4bYq3U76Gj16NGbPno24uDgcPny4wu1VXatXr0bLli1haWkJNze3cl+bjysNb1lZWfD39y93TOk5MZ6enkap81FqtbrCZdevX6/0NV8dNX3NU/3BsEOys3nzZhQXF0vnahiDi4sLGjduLP0D+zh7e3ud2+X9tevi4oJnnnkG//nPf8qdw8PDo1o1lZ5zUdlfuk5OTrCwsMDw4cMxYcKEcsd4e3sDACwtLTFlyhRMmTIFt27dws6dO/H222+jR48euHTpEmxtbatVH/C/vVG///47evToARcXFwDAunXr4OXlVe35SnXv3h3x8fE4cOAAfvvtN7z77rsAgG7duiE1NRXZ2dlo2LChdDIuoP/zX91tbSyenp4ICwvD7Nmz0aJFCwQHBxtl3pYtW1b7H+7w8HCsWLECGzduxIwZM8ods3HjRp1zokr3Lmo0Gp1xj4dDfZSe9FzestJw+ej6Sv8QqO76auM1T+aJYYdk5eLFi5g2bRocHR0xduxYo80bGRmJ5ORkFBcXo0OHDgbPsWXLFvj4+MDJyanGNfn5+cHHxwcrV67ElClTdH7hl7K1tUVoaCiOHDmCZ555psK9FY9r1KgRXn75ZVy5cgUxMTG4cOECWrVqVe0ajx49CgBwdXUFAPTo0QOWlpY4f/48/vnPf1b62Ef3ZD16QjDwMOwoFAq89957aNCggXQicFhYGP7v//4P2dnZ6NKli85eHH2ff2Nsa0NNnToVNjY20knWpjJgwAC0atUKH374IQYOHFjmkOzatWuxY8cOjBs3TtrjUnqF07Fjx9CjRw9p7I8//lhm/sq2LQDs2rULV69elQ5lFRcXY+3atfDx8ZH26jy6vkf3FD569d2j66tqT5uxXvNknhh2qN46fvy4dD5FXl4efv75ZyQkJMDCwgIpKSmVXm1SXYMHD8Y333yD3r17Y9KkSXj++eehVCpx+fJlpKWloV+/fhgwYEClc7z//vtITU1FcHAw3nzzTbRo0QL379/HhQsXsGXLFixfvrzau+c///xz9OnTBx07dsTkyZPRrFkzXLx4Edu3b5euJvv000/xwgsv4MUXX8S//vUvNG/eHIWFhTh37hx++ukn6Wq0Pn36ICAgAO3bt0eTJk2QnZ2NRYsWwcvLC76+vlXWcvHiRen8nzt37iAjIwNz586Fl5cXBg4cCODhP1Dvv/8+3nnnHfz111/o2bMnnJyccPXqVRw4cAB2dnbSZfqlhyHnzZuHXr16wcLCQgpsrq6uCAgIwI4dOxAaGir9BR4WFoYbN27gxo0bWLhwoUHPvzG2taEiIiIQERGh19ji4mKsW7euzHI7OzvpPCRDWVhYYP369QgPD0enTp0wdepUdOrUCRqNBj/99BNWrFiBkJAQLFiwQHqMWq1GWFgY5s6dCycnJ3h5eWHXrl3YsGFDmfkr27bAw71r3bp1w3vvvSddjfXnn3/qXH7eu3dvODs7Izo6Gu+//z4sLS2RmJiIS5culbu+5ORkrF27Fv/4xz9gbW2NwMDAGr/mqR4x9RnSRNVVerVF6Y+VlZVwdXUVISEhIi4uTuTl5ZV5TE2vxhJCCK1WKz7++GPRpk0bYW1tLRo2bCj8/f3F2LFjxdmzZ6VxXl5e4qWXXiq39v/+97/izTffFN7e3kKpVApnZ2cRFBQk3nnnHekTbSv7kD4AYtasWTrLMjIyRK9evYSjo6NQqVTCx8dHTJ48WWdMVlaWeO2118RTTz0llEqlaNKkiQgODhZz5syRxixYsEAEBwcLFxcXYWVlJZo1ayaio6PFhQsXyu3l0bnx2FVY1tbWws/PT8TExIicnJwyj9m4caMIDQ0VDg4OQqVSCS8vL/Hyyy+LnTt3SmM0Go14/fXXRZMmTYRCoShzhc3kyZMFAPGf//xHZ25fX18BQPpE7Efp8/wLUfNtHRISoteH2OH/X41VmYquxnr8OS/98fLyEkLod9ViVa5duyZmzJgh/P39pefh+eefF0uWLBFFRUVlxufk5IiXX35ZODs7C0dHRzFs2DBx6NChMldjVbZtS5+TpUuXCh8fH6FUKoW/v7/45ptvyqzvwIEDIjg4WNjZ2YmnnnpKzJo1S3z11Vdlnq8LFy6IiIgIYW9vr/McGfqap/pHIYSRL1shIiIykEKhwIQJE7BkyRJTl0IywkvPiYiISNYYdoiIiEjWeIIyERGZDZ5ZQbWBe3aIiIhI1hh2iIiISNYYdoiIiEjWeM4OgJKSEvz999+wt7evlS+1IyIiIuMTQqCwsBAeHh6Vfrceww6Av//+u1a+zI6IiIhq36VLlyr9BHqGHfzvi/0uXboEBwcHE1dTMa1Wix07diAiIqLCb26WA/YpP09Kr+xTXtin+SsoKICnp2eVX9DLsIP/fUO1g4OD2YcdW1tbODg41LsXZHWwT/l5Unpln/LCPuuPqk5B4QnKREREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGsMO0RERCRrDDtEREQka5amLoCIqCKhoVWPSUur/TqIqH7jnh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWThp1ly5bhmWeegYODAxwcHNCpUyds3bpVul8IgdjYWHh4eMDGxgZdu3bFiRMndObQaDSYOHEiXFxcYGdnh759++Ly5ct13QoRERGZKZOGnaZNm+LDDz/EoUOHcOjQIXTr1g39+vWTAs38+fOxcOFCLFmyBAcPHoRarUZ4eDgKCwulOWJiYpCSkoLk5GTs378ft2/fRmRkJIqLi03VFhEREZkRk4adPn36oHfv3vDz84Ofnx/+85//oGHDhvj1118hhMCiRYvwzjvvYODAgQgICMCqVatw9+5dJCUlAQDy8/MRHx+PBQsWICwsDG3btsWaNWvwxx9/YOfOnaZsjYiIiMyEpakLKFVcXIzvv/8ed+7cQadOnZCVlYXc3FxERERIY1QqFUJCQpCeno6xY8ciMzMTWq1WZ4yHhwcCAgKQnp6OHj16lLsujUYDjUYj3S4oKAAAaLVaaLXaWuqw5kprM+cajYF9yo+hvVpZ6TO3IRXVjidlm7JPeanPfepbs8nDzh9//IFOnTrh/v37aNiwIVJSUtCqVSukp6cDANzc3HTGu7m5ITs7GwCQm5sLKysrODk5lRmTm5tb4Trnzp2L2bNnl1m+Y8cO2Nra1rSlWpeammrqEuoE+5Sf6vY6fnzVY7ZsMbCYWvSkbFP2KS/1sc+7d+/qNc7kYadFixY4evQobt26hfXr12PkyJHYu3evdL9CodAZL4Qos+xxVY2ZOXMmpkyZIt0uKCiAp6cnIiIi4ODgYGAntU+r1SI1NRXh4eFQKpWmLqfWsE/5MbTXyMiqx2zaVIPCjOxJ2absU17qc5+lR2aqYvKwY2VlhaeffhoA0L59exw8eBCffvop3nrrLQAP9964u7tL4/Py8qS9PWq1GkVFRbh586bO3p28vDwEBwdXuE6VSgWVSlVmuVKprBcbur7UWVPss/4KDdW9bWX1cC/NgAFKFBU97DUtrep5ioqqHmOOT50ct2l52Ke81Mc+9a3X7D5nRwgBjUYDb29vqNVqnd1qRUVF2Lt3rxRkgoKCoFQqdcbk5OTg+PHjlYYdIiIienKYdM/O22+/jV69esHT0xOFhYVITk7Gnj17sG3bNigUCsTExCAuLg6+vr7w9fVFXFwcbG1tMWTIEACAo6MjoqOjMXXqVDRu3BjOzs6YNm0aAgMDERYWZsrWiIiIyEyYNOxcvXoVw4cPR05ODhwdHfHMM89g27ZtCA8PBwBMnz4d9+7dw/jx43Hz5k106NABO3bsgL29vTTHJ598AktLS0RFReHevXvo3r07EhMTYWFhYaq2iIiIyIyYNOzEx8dXer9CoUBsbCxiY2MrHGNtbY3Fixdj8eLFRq6OiIiI5MDsztkhIiIiMiaGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNUtTF0BE9UtoqKkrICKqHu7ZISIiIllj2CEiIiJZY9ghIiIiWWPYISIiIllj2CEiIiJZY9ghIiIiWeOl50RmTp9LvdPSar8OIqL6int2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNZMGnbmzp2L5557Dvb29nB1dUX//v1x+vRpnTGjRo2CQqHQ+enYsaPOGI1Gg4kTJ8LFxQV2dnbo27cvLl++XJetEBERkZkyadjZu3cvJkyYgF9//RWpqal48OABIiIicOfOHZ1xPXv2RE5OjvSzZcsWnftjYmKQkpKC5ORk7N+/H7dv30ZkZCSKi4vrsh0iIiIyQyb9ItBt27bp3E5ISICrqysyMzPRpUsXablKpYJarS53jvz8fMTHx+Prr79GWFgYAGDNmjXw9PTEzp070aNHj9prgIiIiMyeWZ2zk5+fDwBwdnbWWb5nzx64urrCz88PY8aMQV5ennRfZmYmtFotIiIipGUeHh4ICAhAenp63RROREREZsuke3YeJYTAlClT8MILLyAgIEBa3qtXL7zyyivw8vJCVlYW3nvvPXTr1g2ZmZlQqVTIzc2FlZUVnJycdOZzc3NDbm5uuevSaDTQaDTS7YKCAgCAVquFVquthe6Mo7Q2c67RGNinLisrfeYyRkX60aeexymVWp3/AvrVbG69V4WvXXlhn+ZP35oVQghRy7XoZcKECdi8eTP279+Ppk2bVjguJycHXl5eSE5OxsCBA5GUlITRo0frhBcACA8Ph4+PD5YvX15mjtjYWMyePbvM8qSkJNja2ta8GSIiIqp1d+/exZAhQ5Cfnw8HB4cKx5nFnp2JEyfixx9/xL59+yoNOgDg7u4OLy8vnD17FgCgVqtRVFSEmzdv6uzdycvLQ3BwcLlzzJw5E1OmTJFuFxQUwNPTExEREZU+Waam1WqRmpqK8PBwKJVKU5dTa9inrsjIqufatMmIhVVBn3oep1RqMWZMKr78Mhxa7cNe9am5Lns3xrr42pUX9mn+So/MVMWkYUcIgYkTJyIlJQV79uyBt7d3lY+5fv06Ll26BHd3dwBAUFAQlEolUlNTERUVBeDh3p/jx49j/vz55c6hUqmgUqnKLFcqlfViQ9eXOmuKfT5UVKTPHEYsqAr61FMRrVaJoqKHxepTc132bsx18bUrL+zTfOlbr0nDzoQJE5CUlIQffvgB9vb20jk2jo6OsLGxwe3btxEbG4t//vOfcHd3x4ULF/D222/DxcUFAwYMkMZGR0dj6tSpaNy4MZydnTFt2jQEBgZKV2cRERHRk8ukYWfZsmUAgK5du+osT0hIwKhRo2BhYYE//vgDq1evxq1bt+Du7o7Q0FCsXbsW9vb20vhPPvkElpaWiIqKwr1799C9e3ckJibCwsKiLtshIiIiM2Tyw1iVsbGxwfbt26ucx9raGosXL8bixYuNVRoRERHJhFl9zg4RERGRsTHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkaxZmroAIqoboaFVj0lLq/06iIjqGvfsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkayZNOzMnTsXzz33HOzt7eHq6or+/fvj9OnTOmOEEIiNjYWHhwdsbGzQtWtXnDhxQmeMRqPBxIkT4eLiAjs7O/Tt2xeXL1+uy1aIiIjITJk07OzduxcTJkzAr7/+itTUVDx48AARERG4c+eONGb+/PlYuHAhlixZgoMHD0KtViM8PByFhYXSmJiYGKSkpCA5ORn79+/H7du3ERkZieLiYlO0RURERGbE0pQr37Ztm87thIQEuLq6IjMzE126dIEQAosWLcI777yDgQMHAgBWrVoFNzc3JCUlYezYscjPz0d8fDy+/vprhIWFAQDWrFkDT09P7Ny5Ez169KjzvoiIiMh8GLRnJysry9h1AADy8/MBAM7OztJ6cnNzERERIY1RqVQICQlBeno6ACAzMxNarVZnjIeHBwICAqQxRERE9OQyaM/O008/jS5duiA6Ohovv/wyrK2ta1yIEAJTpkzBCy+8gICAAABAbm4uAMDNzU1nrJubG7Kzs6UxVlZWcHJyKjOm9PGP02g00Gg00u2CggIAgFarhVarrXEvtaW0NnOu0RjYpy4rK33mqnpMXc7zOKVSq/NfY67LWC8TY6yLr115YZ/mT9+aFUIIUd3Jjx8/jpUrV+Kbb76BRqPBoEGDEB0djeeff77ahZaaMGECNm/ejP3796Np06YAgPT0dHTu3Bl///033N3dpbFjxozBpUuXsG3bNiQlJWH06NE64QUAwsPD4ePjg+XLl5dZV2xsLGbPnl1meVJSEmxtbQ3ugYiIiOrO3bt3MWTIEOTn58PBwaHCcQbt2QkICMDChQsxf/58/PTTT0hMTMQLL7wAX19fREdHY/jw4WjSpIne802cOBE//vgj9u3bJwUdAFCr1QAe7r15NOzk5eVJe3vUajWKiopw8+ZNnb07eXl5CA4OLnd9M2fOxJQpU6TbBQUF8PT0RERERKVPlqlptVqkpqYiPDwcSqXS1OXUGvapKzKy6rk2bap6TF3O8zilUosxY1Lx5Zfh0GqVRl2XPvPowxjr4mtXXtin+Ss9MlOVGp2gbGlpiQEDBqB3795YunQpZs6ciWnTpmHmzJkYNGgQ5s2bpxNSHieEwMSJE5GSkoI9e/bA29tb535vb2+o1Wqkpqaibdu2AICioiLs3bsX8+bNAwAEBQVBqVQiNTUVUVFRAICcnBwcP34c8+fPL3e9KpUKKpWqzHKlUlkvNnR9qbOm2OdDRUX6zFH1mLqcpyJarRJFRUqjrstYLxFjrouvXXlhn+ZL33prdOn5oUOHMH78eLi7u2PhwoWYNm0azp8/j927d+PKlSvo169fpY+fMGEC1qxZg6SkJNjb2yM3Nxe5ubm4d+8eAEChUCAmJgZxcXFISUnB8ePHMWrUKNja2mLIkCEAAEdHR0RHR2Pq1KnYtWsXjhw5gmHDhiEwMFC6OouIiIieXAbt2Vm4cCESEhJw+vRp9O7dG6tXr0bv3r3RoMHD7OTt7Y0vvvgC/v7+lc6zbNkyAEDXrl11lickJGDUqFEAgOnTp+PevXsYP348bt68iQ4dOmDHjh2wt7eXxn/yySewtLREVFQU7t27h+7duyMxMREWFhaGtEdEREQyYlDYWbZsGV577TWMHj1aOq/mcc2aNUN8fHyl8+hzbrRCoUBsbCxiY2MrHGNtbY3Fixdj8eLFVc5HRERETxaDws7Zs2erHGNlZYWRI0caMj0RERGR0Rh0zk5CQgK+//77Msu///57rFq1qsZFERERERmLQWHnww8/hIuLS5nlrq6uiIuLq3FRRERERMZiUNjJzs4uc5k4AHh5eeHixYs1LoqIiIjIWAwKO66urjh27FiZ5b///jsaN25c46KIiIiIjMWgsDN48GC8+eabSEtLQ3FxMYqLi7F7925MmjQJgwcPNnaNRERERAYz6GqsOXPmIDs7G927d4el5cMpSkpKMGLECJ6zQ0R6CQ01dQVE9KQwKOxYWVlh7dq1+OCDD/D777/DxsYGgYGB8PLyMnZ9RERERDVSo+/G8vPzg5+fn7FqISIiIjI6g8JOcXExEhMTsWvXLuTl5aGkpETn/t27dxulOCKqWzy0RERyZFDYmTRpEhITE/HSSy8hICAACoXC2HURERERGYVBYSc5ORnfffcdevfubex6iIiqRZ+9UWlptV8HEZkvgy49t7KywtNPP23sWoiIiIiMzqCwM3XqVHz66ad6fWs5ERERkSkZdBhr//79SEtLw9atW9G6dWsolUqd+zds2GCU4oiIiIhqyqCw06hRIwwYMMDYtRAREREZnUFhJyEhwdh1EBEREdUKg87ZAYAHDx5g586d+OKLL1BYWAgA+Pvvv3H79m2jFUdERERUUwbt2cnOzkbPnj1x8eJFaDQahIeHw97eHvPnz8f9+/exfPlyY9dJREREZBCD9uxMmjQJ7du3x82bN2FjYyMtHzBgAHbt2mW04oiIiIhqyuCrsX755RdYWVnpLPfy8sKVK1eMUhgRERGRMRi0Z6ekpATFxcVlll++fBn29vY1LoqIiIjIWAwKO+Hh4Vi0aJF0W6FQ4Pbt25g1axa/QoKIiIjMikGHsT755BOEhoaiVatWuH//PoYMGYKzZ8/CxcUF3377rbFrJCIiIjKYQWHHw8MDR48exbfffovDhw+jpKQE0dHRGDp0qM4Jy0RERESmZlDYAQAbGxu89tpreO2114xZDxEREZFRGRR2Vq9eXen9I0aMMKgYIiJzFhpa+f1WVsD48XVTCxHpz6CwM2nSJJ3bWq0Wd+/ehZWVFWxtbRl2iIiIyGwYdDXWzZs3dX5u376N06dP44UXXuAJykRERGRWDP5urMf5+vriww8/LLPXh4iIiMiUDD5BuTwWFhb4+++/jTklkexFRgJFRaaugohIvgwKOz/++KPObSEEcnJysGTJEnTu3NkohREREREZg0Fhp3///jq3FQoFmjRpgm7dumHBggXGqIuIiIjIKAwKOyUlJcaug4iIiKhWGO0EZSIiIiJzZNCenSlTpug9duHChYasgoiIiMgoDAo7R44cweHDh/HgwQO0aNECAHDmzBlYWFigXbt20jiFQmGcKomIiIgMZFDY6dOnD+zt7bFq1So4OTkBePhBg6NHj8aLL76IqVOnGrVIIiIiIkMZdM7OggULMHfuXCnoAICTkxPmzJnDq7GIiIjIrBgUdgoKCnD16tUyy/Py8lBYWFjjooiIiIiMxaCwM2DAAIwePRrr1q3D5cuXcfnyZaxbtw7R0dEYOHCgsWskIiIiMphB5+wsX74c06ZNw7Bhw6DVah9OZGmJ6OhofPTRR0YtkIiopkJDTV0BEZmSQWHH1tYWS5cuxUcffYTz589DCIGnn34adnZ2xq6PiIiIqEZq9KGCOTk5yMnJgZ+fH+zs7CCEqNbj9+3bhz59+sDDwwMKhQIbN27UuX/UqFFQKBQ6Px07dtQZo9FoMHHiRLi4uMDOzg59+/bF5cuXa9IWUZVCQ6v+ISIi82BQ2Ll+/Tq6d+8OPz8/9O7dGzk5OQCA119/vVqXnd+5cwdt2rTBkiVLKhzTs2dPKVTl5ORgy5YtOvfHxMQgJSUFycnJ2L9/P27fvo3IyEgUFxcb0hoRERHJjEGHsSZPngylUomLFy+iZcuW0vJBgwZh8uTJel9+3qtXL/Tq1avSMSqVCmq1utz78vPzER8fj6+//hphYWEAgDVr1sDT0xM7d+5Ejx499OyIiIiI5MqgPTs7duzAvHnz0LRpU53lvr6+yM7ONkphpfbs2QNXV1f4+flhzJgxyMvLk+7LzMyEVqtFRESEtMzDwwMBAQFIT083ah1ERERUPxm0Z+fOnTuwtbUts/zatWtQqVQ1LqpUr1698Morr8DLywtZWVl477330K1bN2RmZkKlUiE3NxdWVlY6H24IAG5ubsjNza1wXo1GA41GI90uKCgAAGi1WunqMnNUWps512gM9aFPK6uqx1RVfml/SmXN+9TnqdKn5tpS2qMxejVnpf2Z82vXGOrDe9QY2Kf507dmhajuWcUAXnrpJbRr1w4ffPAB7O3tcezYMXh5eWHw4MEoKSnBunXrql2wQqFASkoK+vfvX+GYnJwceHl5ITk5GQMHDkRSUhJGjx6tE1wAIDw8HD4+Pli+fHm588TGxmL27NllliclJZUb4oiIiMj83L17F0OGDEF+fj4cHBwqHGfQnp2PPvoIXbt2xaFDh1BUVITp06fjxIkTuHHjBn755ReDi66Ku7s7vLy8cPbsWQCAWq1GUVERbt68qbN3Jy8vD8HBwRXOM3PmTJ1vbi8oKICnpyciIiIqfbJMTavVIjU1FeHh4VAqlaYup9bUhz4jI6ses2lT5feX9vnll+HQamvWZ1XrAvSrubYolVqMGWOcXs1ZaZ/m/No1hvrwHjUG9mn+So/MVMWgsNOqVSscO3YMy5Ytg4WFBe7cuYOBAwdiwoQJcHd3N2RKvVy/fh2XLl2S1hEUFASlUonU1FRERUUBeLj35/jx45g/f36F86hUqnIPtymVynqxoetLnTVlzn0WFVU9Rt/StVoliopq1qc+69Kn5tpmjF7rA3N+7RoT+5SX+tinvvVWO+yUnhD8xRdflHsoqDpu376Nc+fOSbezsrJw9OhRODs7w9nZGbGxsfjnP/8Jd3d3XLhwAW+//TZcXFwwYMAAAICjoyOio6MxdepUNG7cGM7Ozpg2bRoCAwOlq7OIiIjoyVbtsKNUKnH8+HEoFIoar/zQoUMIfeTT10oPLY0cORLLli3DH3/8gdWrV+PWrVtwd3dHaGgo1q5dC3t7e+kxn3zyCSwtLREVFYV79+6he/fuSExMhIWFRY3rIyIiovrPoMNYI0aMQHx8PD788MMarbxr166Vfury9u3bq5zD2toaixcvxuLFi2tUCxEREcmTQWGnqKgIX331FVJTU9G+ffsy34m1cOFCoxRHREREVFPVCjt//fUXmjdvjuPHj6Ndu3YAgDNnzuiMMcbhLSIiIiJjqVbY8fX1RU5ODtLS0gA8/HqIzz77DG5ubrVSHBEREVFNVevrIh4/v2br1q24c+eOUQsiIiIiMiaDvhurlAEfvkxERERUp6oVdhQKRZlzcniODhEREZmzap2zI4TAqFGjpE8fvn//PsaNG1fmaqwNGzYYr0IiIiKiGqhW2Bk5cqTO7WHDhhm1GCIiIiJjq1bYSUhIqK06iIiIiGpFjU5QJiIiIjJ3DDtEREQkaww7REREJGsMO0RERCRrDDtEREQkaww7REREJGvVuvSciMxTaKipKyAiMl/cs0NERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxq+LICIyQ/p8BUhaWu3XQSQH3LNDREREssawQ0RERLLGsENERESyxnN2iIiMLDISKCqq+H6ea0NUt7hnh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI2foEz0GH2+bZqIiOoP7tkhIiIiWWPYISIiIlkzadjZt28f+vTpAw8PDygUCmzcuFHnfiEEYmNj4eHhARsbG3Tt2hUnTpzQGaPRaDBx4kS4uLjAzs4Offv2xeXLl+uwCyIiIjJnJg07d+7cQZs2bbBkyZJy758/fz4WLlyIJUuW4ODBg1Cr1QgPD0dhYaE0JiYmBikpKUhOTsb+/ftx+/ZtREZGori4uK7aICIiIjNm0hOUe/XqhV69epV7nxACixYtwjvvvIOBAwcCAFatWgU3NzckJSVh7NixyM/PR3x8PL7++muEhYUBANasWQNPT0/s3LkTPXr0qLNeiIiIyDyZ7Tk7WVlZyM3NRUREhLRMpVIhJCQE6enpAIDMzExotVqdMR4eHggICJDGEBER0ZPNbC89z83NBQC4ubnpLHdzc0N2drY0xsrKCk5OTmXGlD6+PBqNBhqNRrpdUFAAANBqtdBqtUapvzaU1mbONRqDqfu0sjLOPFWVX9qfUinv7Qn8r0e596pvn/q8tPV5HZrqV4Gp36N1hX2aP31rNtuwU0qhUOjcFkKUWfa4qsbMnTsXs2fPLrN8x44dsLW1NazQOpSammrqEuqEqfocP94482zZot+4MWOejO0JPDm9VtWnPq8NfV6H+r7Gagt/F8lLfezz7t27eo0z27CjVqsBPNx74+7uLi3Py8uT9vao1WoUFRXh5s2bOnt38vLyEBwcXOHcM2fOxJQpU6TbBQUF8PT0REREBBwcHIzditFotVqkpqYiPDwcSqXS1OXUGlP3GRlpnHk2bar8/tI+v/wyHFqtfLcn8HBPx5gx8u9V3z6rem0A+r0O9ZmnNpj6PVpX2Kf5Kz0yUxWzDTve3t5Qq9VITU1F27ZtAQBFRUXYu3cv5s2bBwAICgqCUqlEamoqoqKiAAA5OTk4fvw45s+fX+HcKpUKKpWqzHKlUlkvNnR9qbOmTNVnUZFx5nnkVLJyWVk9/Otdq1WiqEj+2xN4cnqtqk99Xtb6vA5N/WuAv4vkpT72qW+9Jg07t2/fxrlz56TbWVlZOHr0KJydndGsWTPExMQgLi4Ovr6+8PX1RVxcHGxtbTFkyBAAgKOjI6KjozF16lQ0btwYzs7OmDZtGgIDA6Wrs4iIiOjJZtKwc+jQIYQ+8kVEpYeWRo4cicTEREyfPh337t3D+PHjcfPmTXTo0AE7duyAvb299JhPPvkElpaWiIqKwr1799C9e3ckJibCwsKizvshIiIi82PSsNO1a1cIISq8X6FQIDY2FrGxsRWOsba2xuLFi7F48eJaqJCIiIjqO7P9nB0iIiIiY2DYISIiIlkz26uxiIjk6pFTFYmoDnDPDhEREckaww4RERHJGsMOERERyRrDDhEREckaww4RERHJGsMOERERyRrDDhEREckaww4RERHJGsMOERERyRo/QZlkQ59PpU1Lq/06iIjIvHDPDhEREckaww4RERHJGg9jERE94XgImOSOe3aIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYsTV0AUV0KDTV1BUREVNe4Z4eIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjZ+zQ0RUT+nzuVFpabVfB5G5454dIiIikjWzDjuxsbFQKBQ6P2q1WrpfCIHY2Fh4eHjAxsYGXbt2xYkTJ0xYMREREZkbsw47ANC6dWvk5ORIP3/88Yd03/z587Fw4UIsWbIEBw8ehFqtRnh4OAoLC01YMREREZkTsw87lpaWUKvV0k+TJk0APNyrs2jRIrzzzjsYOHAgAgICsGrVKty9exdJSUkmrpqIiIjMhdmHnbNnz8LDwwPe3t4YPHgw/vrrLwBAVlYWcnNzERERIY1VqVQICQlBenq6qcolIiIiM2PWV2N16NABq1evhp+fH65evYo5c+YgODgYJ06cQG5uLgDAzc1N5zFubm7Izs6udF6NRgONRiPdLigoAABotVpotVojd2E8pbWZc43GYGifVla1UU3tUSq1Ov+VsyelV3PsU5+3kT7vnUfn4e8ieanPfepbs0IIIWq5FqO5c+cOfHx8MH36dHTs2BGdO3fG33//DXd3d2nMmDFjcOnSJWzbtq3CeWJjYzF79uwyy5OSkmBra1srtRMREZFx3b17F0OGDEF+fj4cHBwqHGfWe3YeZ2dnh8DAQJw9exb9+/cHAOTm5uqEnby8vDJ7ex43c+ZMTJkyRbpdUFAAT09PREREVPpkmZpWq0VqairCw8OhVCpNXU6tMbTPyMhaLKoWKJVajBmTii+/DIdWK9/tCTw5vZpjn5s2VT1Gn/fOo/Pwd5G81Oc+S4/MVKVehR2NRoNTp07hxRdfhLe3N9RqNVJTU9G2bVsAQFFREfbu3Yt58+ZVOo9KpYJKpSqzXKlU1osNXV/qrKnq9llUVIvF1CKtVomiIvlvT+DJ6dWc+tTnLaTPe6e8efi7SF7qY5/61mvWYWfatGno06cPmjVrhry8PMyZMwcFBQUYOXIkFAoFYmJiEBcXB19fX/j6+iIuLg62trYYMmSIqUsnIiIiM2HWYefy5ct49dVXce3aNTRp0gQdO3bEr7/+Ci8vLwDA9OnTce/ePYwfPx43b95Ehw4dsGPHDtjb25u4ciIiIjIXZh12kpOTK71foVAgNjYWsbGxdVMQEVE9o8/3ZxHJndl/zg4RERFRTTDsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsmfUnKBOV4qfAEhGRobhnh4iIiGSNYYeIiIhkjYexiIjIKPQ53JyWVvt1ED2Oe3aIiIhI1hh2iIiISNZ4GIsMxl3WRERUH3DPDhEREckaww4RERHJGsMOERERyRrDDhEREckaww4RERHJGsMOERERyRrDDhEREckaww4RERHJGj9UkGoVP3iQSB4efS9bWQHjxwORkUBRkelqItIX9+wQERGRrHHPDhER1Rnu7SVTYNghk3v8lx93kRMRkTHxMBYRERHJGsMOERERyRrDDhEREckaww4RERHJGsMOERERyRqvxqJy6XN5KBGRqfASdqoO7tkhIiIiWeOeHZnhXztERES6uGeHiIiIZI1hh4iIiGSNYYeIiIhkjefsPIF4pRURmTP+jiJjY9gxA/q+sfkFmURERNXHsENERE+syv7YLP0Dk+o/hh0iIqI6wI8GMR3ZnKC8dOlSeHt7w9raGkFBQfj5559NXRIRERGZAVns2Vm7di1iYmKwdOlSdO7cGV988QV69eqFkydPolmzZiatjSfaERGZhrF+/+pzniT3yJg3WezZWbhwIaKjo/H666+jZcuWWLRoETw9PbFs2TJTl0ZEREQmVu/DTlFRETIzMxEREaGzPCIiAunp6SaqioiIiMxFvT+Mde3aNRQXF8PNzU1nuZubG3Jzc8t9jEajgUajkW7n5+cDAG7cuAGtVmvU+hoYNU5qcffuXQDX0aCB0pgTmxn2KT9PSq/sU1707/P69apn0+ffg+7dqx7z3XdVj6kOrfZhn9evX4dSWf3tGRVV9Rhj11yqsLAQACCEqHygqOeuXLkiAIj09HSd5XPmzBEtWrQo9zGzZs0SAPjDH/7whz/84Y8Mfi5dulRpVqj3e3ZcXFxgYWFRZi9OXl5emb09pWbOnIkpU6ZIt0tKSnDjxg00btwYCoWiVuutiYKCAnh6euLSpUtwcHAwdTm1hn3Kz5PSK/uUF/Zp/oQQKCwshIeHR6Xj6n3YsbKyQlBQEFJTUzFgwABpeWpqKvr161fuY1QqFVQqlc6yRo0a1WaZRuXg4FDvXpCGYJ/y86T0yj7lhX2aN0dHxyrH1PuwAwBTpkzB8OHD0b59e3Tq1AkrVqzAxYsXMW7cOFOXRkRERCYmi7AzaNAgXL9+He+//z5ycnIQEBCALVu2wMvLy9SlERERkYnJIuwAwPjx4zFe5l9iolKpMGvWrDKH4OSGfcrPk9Ir+5QX9ikfCiGqul6LiIiIqP6q9x8qSERERFQZhh0iIiKSNYYdIiIikjWGHSIiIpI1hh0zcvPmTQwfPhyOjo5wdHTE8OHDcevWrUofo1Aoyv356KOPpDFdu3Ytc//gwYNruZuKGdLnqFGjyvTQsWNHnTEajQYTJ06Ei4sL7Ozs0LdvX1y+fLkWO6ladXvVarV46623EBgYCDs7O3h4eGDEiBH4+++/dcaZepsuXboU3t7esLa2RlBQEH7++edKx+/duxdBQUGwtrbGP/7xDyxfvrzMmPXr16NVq1ZQqVRo1aoVUlJSaqt8vVWnzw0bNiA8PBxNmjSBg4MDOnXqhO3bt+uMSUxMLPf9ev/+/dpupVLV6XPPnj3l9vDnn3/qjDPH7QlUr9fyfu8oFAq0bt1aGmOO23Tfvn3o06cPPDw8oFAosHHjxiofU1/fo3ozzjdUkTH07NlTBAQEiPT0dJGeni4CAgJEZGRkpY/JycnR+Vm5cqVQKBTi/Pnz0piQkBAxZswYnXG3bt2q7XYqZEifI0eOFD179tTp4fr16zpjxo0bJ5566imRmpoqDh8+LEJDQ0WbNm3EgwcParOdSlW311u3bomwsDCxdu1a8eeff4qMjAzRoUMHERQUpDPOlNs0OTlZKJVK8eWXX4qTJ0+KSZMmCTs7O5GdnV3u+L/++kvY2tqKSZMmiZMnT4ovv/xSKJVKsW7dOmlMenq6sLCwEHFxceLUqVMiLi5OWFpail9//bVOeipPdfucNGmSmDdvnjhw4IA4c+aMmDlzplAqleLw4cPSmISEBOHg4FDmfWtK1e0zLS1NABCnT5/W6eHR95k5bk8hqt/rrVu3dHq8dOmScHZ2FrNmzZLGmOM23bJli3jnnXfE+vXrBQCRkpJS6fj6+h6tDoYdM3Hy5EkBQOeFk5GRIQCIP//8U+95+vXrJ7p166azLCQkREyaNMlYpdaIoX2OHDlS9OvXr8L7b926JZRKpUhOTpaWXblyRTRo0EBs27bNKLVXl7G26YEDBwQAnV/Iptymzz//vBg3bpzOMn9/fzFjxoxyx0+fPl34+/vrLBs7dqzo2LGjdDsqKkr07NlTZ0yPHj3E4MGDjVR19VW3z/K0atVKzJ49W7qdkJAgHB0djVWiUVS3z9Kwc/PmzQrnNMftKUTNt2lKSopQKBTiwoUL0jJz3KaP0ifs1Nf3aHXwMJaZyMjIgKOjIzp06CAt69ixIxwdHZGenq7XHFevXsXmzZsRHR1d5r5vvvkGLi4uaN26NaZNm4bCwkKj1V4dNelzz549cHV1hZ+fH8aMGYO8vDzpvszMTGi1WkREREjLPDw8EBAQoPfzZ2zG2KYAkJ+fD4VCUeb720yxTYuKipCZmanzPANAREREhT1lZGSUGd+jRw8cOnQIWq220jGm2naG9Pm4kpISFBYWwtnZWWf57du34eXlhaZNmyIyMhJHjhwxWt3VVZM+27ZtC3d3d3Tv3h1paWk695nb9gSMs03j4+MRFhZW5tP5zWmbGqI+vkerSzafoFzf5ebmwtXVtcxyV1fXMt/oXpFVq1bB3t4eAwcO1Fk+dOhQeHt7Q61W4/jx45g5cyZ+//13pKamGqX26jC0z169euGVV16Bl5cXsrKy8N5776Fbt27IzMyESqVCbm4urKys4OTkpPM4Nzc3vZ8/YzPGNr1//z5mzJiBIUOG6HxBn6m26bVr11BcXAw3Nzed5ZU9z7m5ueWOf/DgAa5duwZ3d/cKx5hq2xnS5+MWLFiAO3fuICoqSlrm7++PxMREBAYGoqCgAJ9++ik6d+6M33//Hb6+vkbtQR+G9Onu7o4VK1YgKCgIGo0GX3/9Nbp37449e/agS5cuACre5qbankDNt2lOTg62bt2KpKQkneXmtk0NUR/fo9XFsFPLYmNjMXv27ErHHDx4EMDDk40fJ4Qod3l5Vq5ciaFDh8La2lpn+ZgxY6T/DwgIgK+vL9q3b4/Dhw+jXbt2es1dldruc9CgQdL/BwQEoH379vDy8sLmzZvLhLvqzGuIutqmWq0WgwcPRklJCZYuXapzX11s08o8Xn9VPZU3/vHl1Z2zLhha07fffovY2Fj88MMPOoG3Y8eOOifWd+7cGe3atcPixYvx2WefGa/waqpOny1atECLFi2k2506dcKlS5fw8ccfS2GnunPWJUPrSkxMRKNGjdC/f3+d5ea6Taurvr5H9cWwU8veeOONKq+Sad68OY4dO4arV6+Wue+///1vmTRdnp9//hmnT5/G2rVrqxzbrl07KJVKnD171mj/MNZVn6Xc3d3h5eWFs2fPAgDUajWKiopw8+ZNnb07eXl5CA4O1ntefdRFr1qtFlFRUcjKysLu3bt19uqUpza2aXlcXFxgYWFR5q+5vLy8CntSq9Xljre0tETjxo0rHVOd14QxGdJnqbVr1yI6Ohrff/89wsLCKh3boEEDPPfcc9LruK7VpM9HdezYEWvWrJFum9v2BGrWqxACK1euxPDhw2FlZVXpWFNvU0PUx/dodfGcnVrm4uICf3//Sn+sra3RqVMn5Ofn48CBA9Jjf/vtN+Tn5+v1j3V8fDyCgoLQpk2bKseeOHECWq0W7u7uNertUXXVZ6nr16/j0qVLUg9BQUFQKpU6h3FycnJw/Phxo4ed2u61NOicPXsWO3fulH7ZVKY2tml5rKysEBQUVOZwWWpqaoU9derUqcz4HTt2oH379lAqlZWOMfa205chfQIP9+iMGjUKSUlJeOmll6pcjxACR48erfXtVhFD+3zckSNHdHowt+0J1KzXvXv34ty5c+WeD/k4U29TQ9TH92i11f050VSRnj17imeeeUZkZGSIjIwMERgYWOYy5RYtWogNGzboLMvPzxe2trZi2bJlZeY8d+6cmD17tjh48KDIysoSmzdvFv7+/qJt27YmuyS7un0WFhaKqVOnivT0dJGVlSXS0tJEp06dxFNPPSUKCgqkx4wbN040bdpU7Ny5Uxw+fFh069bNLC49r06vWq1W9O3bVzRt2lQcPXpU51JWjUYjhDD9Ni29fDc+Pl6cPHlSxMTECDs7O+kKlRkzZojhw4dL40sva508ebI4efKkiI+PL3NZ6y+//CIsLCzEhx9+KE6dOiU+/PBDk1/WWt0+k5KShKWlpfj8888r/EiA2NhYsW3bNnH+/Hlx5MgRMXr0aGFpaSl+++23Ou+vVHX7/OSTT0RKSoo4c+aMOH78uJgxY4YAINavXy+NMcftKUT1ey01bNgw0aFDh3LnNMdtWlhYKI4cOSKOHDkiAIiFCxeKI0eOSFd0yuU9Wh0MO2bk+vXrYujQocLe3l7Y29uLoUOHlrm8E4BISEjQWfbFF18IGxubcj9n5eLFi6JLly7C2dlZWFlZCR8fH/Hmm2+W+YyaulTdPu/evSsiIiJEkyZNhFKpFM2aNRMjR44UFy9e1HnMvXv3xBtvvCGcnZ2FjY2NiIyMLDOmrlW316ysLAGg3J+0tDQhhHls088//1x4eXkJKysr0a5dO7F3717pvpEjR4qQkBCd8Xv27BFt27YVVlZWonnz5uUG8++//160aNFCKJVK4e/vr/OPp6lUp8+QkJByt9vIkSOlMTExMaJZs2bCyspKNGnSRERERIj09PQ67Kh81elz3rx5wsfHR1hbWwsnJyfxwgsviM2bN5eZ0xy3pxDVf+3eunVL2NjYiBUrVpQ7nzlu09KPB6jotSin96i+FEL8/7OQiIiIiGSI5+wQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEJFJKRQKbNy4Ubr9559/omPHjrC2tsazzz5b4TIiMo19+/ahT58+8PDwKPP+rS1XrlzBsGHD0LhxY9ja2uLZZ59FZmam3o9n2CGiWjFq1CgoFAooFAoolUq4ubkhPDwcK1euRElJiTQuJycHvXr1km7PmjULdnZ2OH36NHbt2lXhMiIyjTt37qBNmzZYsmRJnazv5s2b6Ny5M5RKJbZu3YqTJ09iwYIFaNSokd5z8FvPiajW9OzZEwkJCSguLsbVq1exbds2TJo0CevWrcOPP/4IS0tLqNVqncecP38eL730Ery8vCpdVl1FRUVVfmM1EVWtV69eOn+gPK6oqAjvvvsuvvnmG9y6dQsBAQGYN28eunbtatD65s2bB09PTyQkJEjLmjdvXq05uGeHiGqNSqWCWq3GU089hXbt2uHtt9/GDz/8gK1btyIxMRGA7mEshUKBzMxMvP/++1AoFIiNjS13GfBwt/agQYPg5OSExo0bo1+/frhw4YK07lGjRqF///6YO3cuPDw84OfnV63Hffzxx3B3d0fjxo0xYcIEaLVaaYxGo8H06dPh6ekJlUoFX19fxMfHS/efPHkSvXv3RsOGDeHm5obhw4fj2rVrtfIcE5mb0aNH45dffkFycjKOHTuGV155BT179sTZs2cNmu/HH39E+/bt8corr8DV1RVt27bFl19+Wa05GHaIqE5169YNbdq0wYYNG8rcl5OTg9atW2Pq1KnIycnBtGnTyl129+5dhIaGomHDhti3bx/279+Phg0bomfPnigqKpLm27VrF06dOoXU1FRs2rRJ78elpaXh/PnzSEtLw6pVq5CYmCiFMwAYMWIEkpOT8dlnn+HUqVNYvnw5GjZsKPUQEhKCZ599FocOHcK2bdtw9epVREVF1d6TSmQmzp8/j2+//Rbff/89XnzxRfj4+GDatGl44YUXdPbMVMdff/2FZcuWwdfXF9u3b8e4cePw5ptvYvXq1XrPwcNYRFTn/P39cezYsTLL1Wo1LC0t0bBhQ+nwVsOGDcssW7lyJRo0aICvvvoKCoUCAJCQkIBGjRphz549iIiIAADY2dnhq6++kg5f6fs4JycnLFmyBBYWFvD398dLL72EXbt2YcyYMThz5gy+++47pKamIiwsDADwj3/8Q+ph2bJlaNeuHeLi4qRlK1euhKenJ86cOSPtYSKSo8OHD0MIUeZ1rtFo0LhxYwDAhQsX4O3tXek8EyZMkM4JKikpQfv27aX3VNu2bXHixAksW7YMI0aM0Ksuhh0iqnNCCClsGCIzMxPnzp2Dvb29zvL79+/j/Pnz0u3AwECd83T0fVzr1q1hYWEh3XZ3d8cff/wBADh69CgsLCwQEhJSYW1paWnSnp5HnT9/nmGHZK2kpAQWFhbIzMzUeQ8BkN4TTz31FE6dOlXpPE5OTtL/u7u7o1WrVjr3t2zZEuvXr9e7LoYdIqpzp06dqvIvu8qUlJQgKCgI33zzTZn7mjRpIv2/nZ2dQY9TKpU69ykUCukKMhsbmypr69OnD+bNm1fmPnd390ofS1TftW3bFsXFxcjLy8OLL75Y7hilUgl/f3+95+zcuTNOnz6ts+zMmTPVumCBYYeI6tTu3bvxxx9/YPLkyQbP0a5dO6xduxaurq5wcHCo9cc9KjAwECUlJdi7d690GOvxdaxfvx7NmzeHpSV/xZL83L59G+fOnZNuZ2Vl4ejRo3B2doafnx+GDh2KESNGYMGCBWjbti2uXbuG3bt3IzAwEL179672+iZPnozg4GDExcUhKioKBw4cwIoVK7BixQq95+AJykRUazQaDXJzc3HlyhUcPnwYcXFx6NevHyIjI/U+1l6eoUOHwsXFBf369cPPP/+MrKws7N27F5MmTcLly5eN/rhHNW/eHCNHjsRrr72GjRs3IisrC3v27MF3330H4OG5Bjdu3MCrr76KAwcO4K+//sKOHTvw2muvobi42OCeiczFoUOH0LZtW7Rt2xYAMGXKFLRt2xb//ve/ATw8D27EiBGYOnUqWrRogb59++K3336Dp6enQet77rnnkJKSgm+//RYBAQH44IMPsGjRIgwdOlTvOfhnBxHVmm3btsHd3R2WlpZwcnJCmzZt8Nlnn2HkyJFo0MDwv7VsbW2xb98+vPXWWxg4cCAKCwvx1FNPoXv37pXusTH0cY9btmwZ3n77bYwfPx7Xr19Hs2bN8PbbbwMAPDw88Msvv+Ctt95Cjx49oNFo4OXlhZ49e9aoZyJz0bVrVwghKrxfqVRi9uzZmD17ttHWGRkZicjISIMfrxCVVUxERERUz/HPDCIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikrX/BzG+SOTUJuwaAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "differences = hf_model.model.layers[0].block_sparse_moe(test_tensor)[0] - tl_model.blocks[0].mlp(test_tensor)\n",
    "\n",
    "# Flatten the differences to create a one-dimensional tensor\n",
    "flattened_differences = differences.flatten().cpu().detach().numpy()\n",
    "\n",
    "# Plot the histogram of the differences\n",
    "plt.hist(flattened_differences, bins=50, alpha=0.75, color='blue')\n",
    "plt.title('Differences Between MLP Outputs')\n",
    "plt.xlabel('Difference')\n",
    "plt.ylabel('Frequency')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ac306e1c-9972-466a-8f4a-f3eb56042f53",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.46239426732063293"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].block_sparse_moe(test_tensor)[0][0, 0, 0].item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e9481397-6e87-435a-a0cf-ef409630d17c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4623942971229553"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].mlp(test_tensor)[0, 0, 0].item()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8176dc01-375b-4b48-b9f0-10efc4548eaf",
   "metadata": {},
   "source": [
    "## Compare Attention Outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "5172efa2-0066-4ae0-a6a2-530d815b053b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.3395,  0.2164, -0.0300,  ...,  0.1450,  0.0525, -0.1044]]],\n",
       "       grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tl_model.blocks[0].attn.forward(test_tensor, test_tensor, test_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "92781a06-e16d-43f9-be4c-3ef04b3d4b08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.3395,  0.2164, -0.0300,  ...,  0.1450,  0.0525, -0.1044]]],\n",
       "       grad_fn=<UnsafeViewBackward0>)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hf_model.model.layers[0].self_attn.forward(test_tensor)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "943cd506-2bb8-45bf-afc7-7f6b4f8043f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[False, False, False,  ..., False, False, False]]])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(tl_model.blocks[0].attn.forward(test_tensor, test_tensor, test_tensor) == \n",
    " hf_model.model.layers[0].self_attn.forward(test_tensor)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "57ffc181-abed-4784-86eb-6e6b4f174bc5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(236)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.sum(tl_model.blocks[0].attn.forward(test_tensor, test_tensor, test_tensor) == \n",
    " hf_model.model.layers[0].self_attn.forward(test_tensor)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "7427fd15-3029-45c3-9d12-64c80f1048f1",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAHFCAYAAAAUpjivAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABJnUlEQVR4nO3de1xU5d7///cIw1E8gYIkIhlqJqVpWWapKXjCSis7WGqa27Zmknrbbd7fHbYN09Js484O21N56myHuxTMQ5l2a5SZ1tYyjwmRSuIRRrh+f/hjdiMgMAMOLF/Px4OHrmuutdZnXTPAm2vWmmUzxhgBAABYVC1vFwAAAFCVCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDuoMgsXLpTNZnN+BQQEKCIiQt26ddO0adOUnZ1dbJ3k5GTZbDaXtvz8fD3yyCNq3LixfHx81LZtW0nS0aNHde+996pRo0ay2Wy64447LsJR1Vx79+51eT5sNpvq1Kmja665RrNnz1ZBQYFb2/3kk0+UnJxcucVWU4cPH5a/v79sNpu+/vrrEvukpKRoxYoVxdp/+OEHJScna+/evVVbZBl1rFu3TjabTevWrbsodZzPGKOlS5fq1ltvVf369eXv76/LL79co0eP1oEDB9ze7qFDh5ScnKytW7dWXrEXsHHjRiUnJ+uPP/64KPuDhwxQRRYsWGAkmQULFphNmzaZzz//3LzzzjsmKSnJ1K1b1zRo0MCkp6e7rHPgwAGzadMml7bZs2cbSSY1NdVs3LjRbNu2zRhjTFJSkvHz8zOLFy82mzZtMjt37rxox1YT7dmzx0gyY8aMMZs2bTKbNm0yn376qfnrX/9qJJlx48a5td3Ro0ebS+VHyaxZs4wkI8k88sgjJfYJDg42Q4YMKdb+9ttvG0lm7dq1VVtkGXUcO3bMbNq0yRw7duyi1PFnBQUF5p577jGSzH333WdWrFhh1q5da1588UXTpEkTU69ePbNhwwa3tr1lyxbnz5uL4bnnnjOSzJ49ey7K/uAZXy9lLFxC2rRpow4dOjiX77zzTj3++OPq3LmzBgwYoJ9++knh4eGSpCZNmqhJkyYu62/fvl2BgYF69NFHi7U3b95cgwYNqrRaT58+rcDAwErbXnXUtGlT3XDDDc7lXr16afv27Vq2bJlmzpzpxcqqv/nz56tRo0aKjo7WsmXLNGvWrBr3eqlTp47L838xTZ8+XW+++aaeffZZPfHEE872rl276p577lHHjh1155136t///rfq1avnlRphUd5OW7CuopmdLVu2lPj4W2+9ZSSZKVOmONueeuopl1kC/f9/Rf/5q2i7538V/cWcl5dn/v73v5uWLVsaPz8/ExYWZoYOHWqys7Nd9h8dHW369u1r3n33XdO2bVvj7+9vnnjiCWOMMZmZmeYvf/mLueyyy4zdbjfNmjUzycnJxuFwONcvmil57rnnzMyZM02zZs1McHCwueGGG4rNThljzFdffWUSExNNgwYNjL+/v7n88svN2LFjXfrs2rXL3HfffaZhw4bGz8/PtGrVysyZM8elT0FBgfn73/9uWrRoYQICAkzdunVNXFycmT179gWfjz/Xe77ExETTtGnTYu3Lly83N9xwgwkKCjLBwcEmISHBfPPNN87HhwwZUuJzsWfPHnPXXXeZ1q1bF9uPJPPWW2852zIyMowk8+GHHzrbyjP+xlT8uf70009Nu3btTEBAgGnZsqWZN2/eBcfsz7766isjyYwfP968+uqrRpJ54403XPqUNBZdunQp9TX751mI9PR0c+utt5qQkBATGBhoOnXqZFavXu2y/aLvj+3bt5t7773X1KlTxzRq1Mg89NBD5o8//iizDmOMWbt2bYkzTB988IG54YYbTGBgoKldu7bp0aOH2bhxo1v7L0leXp6pX7++ufLKK01hYWGJfZYuXWokmeeff97ZFh0dXeIMVZcuXYod0/lfTz31lDHm3Os0ODjYbN++3dx6660mKCjIhIWFmdGjR5uTJ086t1n0PVLS7NCft1c0DqX9DPrss89Mly5dTIMGDUxAQICJiooyAwYMcNkXLi7CDqpMWWHnxIkTxsfHx3Tv3t3Zdn7Y2bRpk+nTp48JDAx0vvWSlZVlNm3aZNq1a2cuv/xyZ/uxY8dMQUGB6dWrlwkODjZTpkwx6enp5l//+pe57LLLTOvWrc2pU6ec246OjjaNGzc2l19+uZk/f75Zu3at2bx5s8nMzDRRUVEmOjravPLKK2b16tXm73//u/H39zdDhw51rl/0g7FZs2amV69eZsWKFWbFihUmLi7O1K9f3+WH/8qVK43dbjdXX321WbhwoVmzZo2ZP3++uffee519duzY4Qwur7/+uklLSzPjx483tWrVMsnJyc5+06ZNMz4+Puapp54yn332mVm5cqWZPXu2S5+SFNU7ffp043A4jMPhMIcPHzbz5s0zvr6+ZvLkyS79n3nmGWOz2cywYcPMxx9/bN577z1z4403muDgYLNjxw5jjDE///yzueuuu4wk5/OwadMmc+bMGfPyyy8bSebQoUPGGGMcDofzF/mIESOc+5k+fbrx9fU1ubm5xhhT7vGv6HPdpEkT07p1a/P666+bVatWmbvvvttIMuvXr7/guBUZMWKEkWR27NhhcnNzTVBQkOnatatLn02bNpnAwEDTp08f51js2LHDZGdnm5SUFCPJ/POf/3Q+VhTK3njjDWOz2cwdd9xh3nvvPfPRRx+ZxMRE4+Pj4xJ4ir4/WrZsaf72t7+Z9PR0M2vWLOPv728eeuihMuswpuSws2TJEiPJJCQkmBUrVpg333zTtG/f3vj5+ZkvvviiwvsvycaNG40k5x8UJTl+/LipVauW6dmzp7OtPGHn2LFjzp83//M//+M85gMHDhhjzoUdPz8/07RpU/PMM8+YtLQ0k5ycbHx9fU1iYqJzm+UNOwcOHDBjxowxksx7773n8jNoz549JiAgwMTHx5sVK1aYdevWmSVLlpgHH3zQ5OTkXHCMUHUIO6gyZYUdY4wJDw83V155pXP5/LBjzH/+Kjtfly5dzFVXXeXStmzZMiPJvPvuuy7tRe/nv/TSS8626Oho4+PjU+xcn5EjR5ratWubffv2ubQ///zzzl92xvznB2NcXJw5e/ass9/mzZuNJLNs2TJnW/PmzU3z5s3N6dOnSx2Lnj17miZNmhQ7l+LRRx81AQEB5ujRo8aYc7Mjbdu2LXU7pSmqt6SvoUOHuhzD/v37ja+vrxkzZozLNo4fP24iIiLMwIEDnW2lnbPz888/G0nm9ddfN8YYs2HDBiPJTJw40cTExDj7xcfHm06dOjmXyzv+FX2uAwICXLZ5+vRp06BBAzNy5Mgyx+7kyZOmTp065oYbbnC2DRkyxNhsNvPzzz+79K3oOTsnT540DRo0MP369XNpLygoMNdcc425/vrrnW1F3x8zZsxw6Ttq1CgTEBDgMmNSWh3nh52CggITGRlp4uLiTEFBgbPf8ePHTaNGjVyem4rs/3zLly83kszLL79cah9jiv9MKE/YMebC5+wUzUC++OKLLu3PPPOMkeQ8T6i8YceY0s/Zeeedd4wks3Xr1gseJy4ursaCVxljKnV7H3/8serVq6d+/frp7Nmzzq+2bdsqIiKi2BUoV199tVq0aFFsG926dVNkZKTLNnr37i1JWr9+vUv/vn37ysfHx2WbkrRv3z5J0q5du7R7924NHz5cAQEBJdZ95swZffbZZ+rfv7+CgoJc9tunTx+dOXNGX331lSTp+uuv13fffadRo0Zp1apVys3NrdAYjR07Vlu2bNGWLVu0du1apaSk6K233tJ9993n7LNq1SqdPXtWgwcPdqklICBAXbp0KdeVPM2bN1ezZs20evVqSVJ6erri4uL0wAMPaM+ePdq9e7fy8vK0YcMG9ejRw7leece/os9127Zt1bRpU+dyQECAWrRo4XyeLuStt95Sbm6uhg0b5mwbNmyYjDFasGBBmetfyMaNG3X06FENGTLE5TgKCwvVq1cvbdmyRSdPnnRZ57bbbnNZvvrqq3XmzJkSr3Asy86dO3Xo0CE9+OCDqlXrP78SateurTvvvFNfffWVTp06VWX7P58xptgVmZXl/PP77r//fknS2rVrK20fbdu2lZ+fn/7yl79o0aJF+uWXXypt23AfJyjDa06ePKkjR44oLi6u0rb522+/6Y8//pCfn1+Jjx8+fNhluXHjxiVu46OPPpLdbi/XNkJDQ12W/f39JZ072VmSfv/9d0kqduL1nx05ckRnz55VamqqUlNTL7jfSZMmKTg4WIsXL9bLL78sHx8f3XLLLZo+fbrLieCladKkiUu/rl27ymazadKkSVq1apV69uyp3377TZJ03XXXlbiNP/9SvJDu3btr5cqVkqTVq1crPj5ecXFxCg8P1+rVqxUbG6vTp0+7hJ3yjn9Fn+vznyfp3HNV9DxdyLx58xQQEKBevXo5LzW++uqr1axZMy1cuFBTpkxxCbwVUTTWd911V6l9jh49quDgYOdyWa+5ijhy5Iikkr8XIiMjVVhYqJycHAUFBXm0/6KguWfPnlL7nDx5UocPH1a7du3KfwDl5OvrW6zuiIgISf8Zg8rQvHlzrV69WjNmzNDo0aN18uRJXX755Xrsscc0duzYStsPKoawA6/53//9XxUUFKhr166Vts2wsDCFhoY6f8GeLyQkxGW5pL8gw8LCdPXVV+uZZ54pcRuRkZEVqqlhw4aSpIMHD5bap379+vLx8dGDDz6o0aNHl9gnJiZG0rkf2uPGjdO4ceP0xx9/aPXq1XryySfVs2dPHThwwOWXUnkVzUZ999136tmzp8LCwiRJ77zzjqKjoyu8vSLdu3fXvHnztHnzZv3f//2f/ud//keSdOuttyo9PV379u1T7dq1Xa4OKu/4V/S5dteuXbu0YcMGSXKZGfqzVatWqU+fPm5tv2isU1NTS71KquhqxapQFAAyMzOLPXbo0CHVqlVL9evX93g/7du3V/369fXhhx9q2rRpJX7vffjhhyosLFR8fLyzLSAgQHl5ecX6Hj582Dl25XH27FkdOXLEJfBkZWVJ+s8YFM28nr+/ioahm2++WTfffLMKCgr09ddfKzU1VUlJSQoPD9e9995boW2hchB24BX79+/XhAkTVLduXY0cObLStpuYmKjly5eroKBAHTt2dHsbn3zyiZo3b14pP+RbtGih5s2ba/78+Ro3bpzzr+A/CwoKUrdu3fTtt9/q6quvLnW24nz16tXTXXfdpV9//VVJSUnau3evWrduXeEaiz6IrVGjRpKknj17ytfXV7t379add955wXX//Ff9+Zdhd+/eXTabTf/v//0/1apVS7fccoskqUePHvqv//ov7du3T7fccovLLE55x78ynuvymDdvniTptdde0xVXXOHy2OnTp3X77bdr/vz5zrBT2mxRabMfN910k+rVq6cffvih2McreKK8s1YtW7bUZZddpqVLl2rChAnOEHLy5Em9++67uvHGG90K0Ofz8/PTf/3Xf+nJJ5/Uc889p4kTJ7o8np2drUmTJik8PFwPP/yws71Zs2batm2bS99du3Zp586dLmGnPLNLS5Ys0WOPPeZcXrp0qSQ5/+AKDw9XQEBAsf198MEHxbZVnv35+PioY8eOatWqlZYsWaJvvvmGsOMlhB1Uue3btzvPQ8jOztYXX3yhBQsWyMfHR++//75z5qMy3HvvvVqyZIn69OmjsWPH6vrrr5fdbtfBgwe1du1a3X777erfv/8Ft/H0008rPT1dnTp10mOPPaaWLVvqzJkz2rt3rz755BO9/PLLF3xLqiT//Oc/1a9fP91www16/PHH1bRpU+3fv1+rVq3SkiVLJEkvvviiOnfurJtvvll//etf1axZMx0/flw///yzPvroI61Zs0aS1K9fP+dnFzVs2FD79u3T7NmzFR0drdjY2DJr2b9/v/P8n5MnT2rTpk2aNm2aoqOjNWDAAEnnfsE8/fTTmjx5sn755Rf16tVL9evX12+//abNmzcrODhYU6ZMkSTn25DTp09X79695ePj4wxsjRo1Ups2bZSWlqZu3bo5f2n26NFDR48e1dGjRzVr1iy3xr8ynuuynD17Vq+//rquvPJKl1/Af9avXz99+OGH+v3339WwYUPFxcVp3bp1+uijj9S4cWOFhISoZcuWatOmjSTp1VdfVUhIiAICAhQTE6PQ0FClpqZqyJAhOnr0qO666y41atRIv//+u7777jv9/vvvmjt3boVrL62O89WqVUszZszQoEGDlJiYqJEjRyovL0/PPfec/vjjDz377LMV3ndpnnjiCX333XfOf++55x7VrVtX27Zt03PPPafjx4/r448/Vt26dZ3rPPjgg3rggQc0atQo3Xnnndq3b59mzJhR7OdG8+bNFRgYqCVLlujKK69U7dq1FRkZ6ZwJ9PPz08yZM3XixAldd9112rhxo6ZOnarevXurc+fOks7N9D7wwAOaP3++mjdvrmuuuUabN292hqLzx1c69307ZMgQ2e12tWzZUkuWLNGaNWvUt29fNW3aVGfOnNH8+fMlyeXtWlxkXj5BGhZ2/meL+Pn5mUaNGpkuXbqYlJSUYp+FYoznV2MZc+4S5+eff95cc801JiAgwNSuXdu0atXKjBw50vz000/OfkWfvVKS33//3Tz22GMmJibG2O1206BBA9O+fXszefJkc+LECWPMhT+3RudduWHMucuBe/fuberWrWv8/f1N8+bNzeOPP+7SZ8+ePWbYsGHOz5dp2LCh6dSpk5k6daqzz8yZM02nTp1MWFiY83La4cOHm71795Z4LH/ets67CisgIMC0aNHCJCUlmczMzGLrrFixwnTr1s3UqVPH+Pv7m+joaHPXXXe5XA6dl5dnHn74YdOwYUNjs9mKXaHy+OOPG0nmmWeecdl2bGyskeT8ROw/K8/4G+P5c33+FT0lHb+kC36G0cqVK40kM3PmTGOMMVu3bjU33XSTCQoKcvl8G2POfRp4TEyM8fHxKXbVz/r1603fvn1NgwYNjN1uN5dddpnp27evefvtt519ir4/fv/9d5cair7X/jzupdVR2ufsrFixwnTs2NEEBASY4OBg0717d/Pll1+69KnI/ktTWFholixZYrp27Wrq1atn/Pz8TExMjPnrX/9a7Aq8ov4zZswwl19+uQkICDAdOnQwa9asKfG5W7ZsmWnVqpWx2+0lfs7Otm3bTNeuXU1gYKBp0KCB+etf/+ryejLm3GXsDz/8sAkPDzfBwcGmX79+Zu/evSV+T0+aNMlERkaaWrVqOcd006ZNpn///iY6Otr4+/ub0NBQ06VLF5fPkcLFZzOmki+HAQCgGhk6dKjeeecdnThxwtulwEu49BwAAFgaYQcAAFgab2MBAABLY2YHAABYGmEHAABYGmEHAABYGh8qKKmwsFCHDh1SSEhIld2ADgAAVC5jjI4fP67IyMgL3rOPsKNz93+JiorydhkAAMANBw4cuOAn2xN29J8bBh44cEB16tRxaxsOh0NpaWlKSEgo9W7NuDDG0HOMoWcYP88xhp5jDMsvNzdXUVFRZd74l7Cj/9z5uk6dOh6FnaCgINWpU4cXp5sYQ88xhp5h/DzHGHqOMay4sk5B4QRlAABgaYQdAABgaYQdAABgaYQdAABgaV4PO7/++qseeOABhYaGKigoSG3btlVGRobzcWOMkpOTFRkZqcDAQHXt2lU7duxw2UZeXp7GjBmjsLAwBQcH67bbbtPBgwcv9qEAAIBqyKthJycnRzfddJPsdrs+/fRT/fDDD5o5c6bq1avn7DNjxgzNmjVLc+bM0ZYtWxQREaH4+HgdP37c2ScpKUnvv/++li9frg0bNujEiRNKTExUQUGBF44KAABUJ1699Hz69OmKiorSggULnG3NmjVz/t8Yo9mzZ2vy5MkaMGCAJGnRokUKDw/X0qVLNXLkSB07dkzz5s3TG2+8oR49ekiSFi9erKioKK1evVo9e/a8qMcEAACqF6/O7Hz44Yfq0KGD7r77bjVq1Ejt2rXTa6+95nx8z549ysrKUkJCgrPN399fXbp00caNGyVJGRkZcjgcLn0iIyPVpk0bZx8AAHDp8urMzi+//KK5c+dq3LhxevLJJ7V582Y99thj8vf31+DBg5WVlSVJCg8Pd1kvPDxc+/btkyRlZWXJz89P9evXL9anaP3z5eXlKS8vz7mcm5sr6dwHOTkcDreOpWg9d9cHY1gZGEPPMH6eYww9xxiWX3nHyKthp7CwUB06dFBKSookqV27dtqxY4fmzp2rwYMHO/ud/8mIxpgyPy3xQn2mTZumKVOmFGtPS0tTUFBQRQ/DRXp6ukfrgzGsDIyhZxg/zzGGnmMMy3bq1Kly9fNq2GncuLFat27t0nbllVfq3XfflSRFRERIOjd707hxY2ef7Oxs52xPRESE8vPzlZOT4zK7k52drU6dOpW430mTJmncuHHO5aJ7ayQkJHh0u4j09HTFx8fz8d5uYgw9xxh6hvHzHGPoOcaw/IremSmLV8POTTfdpJ07d7q07dq1S9HR0ZKkmJgYRUREKD09Xe3atZMk5efna/369Zo+fbokqX379rLb7UpPT9fAgQMlSZmZmdq+fbtmzJhR4n79/f3l7+9frN1ut3v8wqqMbVzqGEPPMYaeYfw8xxh6jjEsW3nHx6th5/HHH1enTp2UkpKigQMHavPmzXr11Vf16quvSjr39lVSUpJSUlIUGxur2NhYpaSkKCgoSPfff78kqW7duho+fLjGjx+v0NBQNWjQQBMmTFBcXJzz6iwAAHDp8mrYue666/T+++9r0qRJevrppxUTE6PZs2dr0KBBzj4TJ07U6dOnNWrUKOXk5Khjx45KS0tzuZ37Cy+8IF9fXw0cOFCnT59W9+7dtXDhQvn4+HjjsAAAQDXi1bAjSYmJiUpMTCz1cZvNpuTkZCUnJ5faJyAgQKmpqUpNTa2CCgEAQE3m9bAD4MK6dSu7z9q1VV8HANRUXr83FgAAQFUi7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEsj7AAAAEvz9XYBAC6Obt3K7rN2bdXXAQAXGzM7AADA0gg7AADA0gg7AADA0gg7AADA0gg7AADA0gg7AADA0gg7AADA0gg7AADA0gg7AADA0gg7AADA0gg7AADA0rwadpKTk2Wz2Vy+IiIinI8bY5ScnKzIyEgFBgaqa9eu2rFjh8s28vLyNGbMGIWFhSk4OFi33XabDh48eLEPBQAAVFNen9m56qqrlJmZ6fz6/vvvnY/NmDFDs2bN0pw5c7RlyxZFREQoPj5ex48fd/ZJSkrS+++/r+XLl2vDhg06ceKEEhMTVVBQ4I3DAQAA1YzX73ru6+vrMptTxBij2bNna/LkyRowYIAkadGiRQoPD9fSpUs1cuRIHTt2TPPmzdMbb7yhHj16SJIWL16sqKgorV69Wj179ryoxwIAAKofr4edn376SZGRkfL391fHjh2VkpKiyy+/XHv27FFWVpYSEhKcff39/dWlSxdt3LhRI0eOVEZGhhwOh0ufyMhItWnTRhs3biw17OTl5SkvL8+5nJubK0lyOBxyOBxuHUfReu6uD8awNH5+ZfcpGrILjWFFtnOp4jXoOcbQc4xh+ZV3jGzGGFPFtZTq008/1alTp9SiRQv99ttvmjp1qv79739rx44d2rlzp2666Sb9+uuvioyMdK7zl7/8Rfv27dOqVau0dOlSPfTQQy7BRZISEhIUExOjV155pcT9Jicna8qUKcXaly5dqqCgoMo9SAAAUCVOnTql+++/X8eOHVOdOnVK7efVmZ3evXs7/x8XF6cbb7xRzZs316JFi3TDDTdIkmw2m8s6xphibecrq8+kSZM0btw453Jubq6ioqKUkJBwwcG6EIfDofT0dMXHx8tut7u1jUsdY1iyxMSy+3z88bl/LzSGFdnOpYrXoOcYQ88xhuVX9M5MWbz+NtafBQcHKy4uTj/99JPuuOMOSVJWVpYaN27s7JOdna3w8HBJUkREhPLz85WTk6P69eu79OnUqVOp+/H395e/v3+xdrvd7vELqzK2caljDF3l55fd5/zhKmkM3dnOpYrXoOcYQ88xhmUr7/h4/WqsP8vLy9OPP/6oxo0bKyYmRhEREUpPT3c+np+fr/Xr1zuDTPv27WW32136ZGZmavv27RcMOwAA4NLh1ZmdCRMmqF+/fmratKmys7M1depU5ebmasiQIbLZbEpKSlJKSopiY2MVGxurlJQUBQUF6f7775ck1a1bV8OHD9f48eMVGhqqBg0aaMKECYqLi3NenQUAAC5tXg07Bw8e1H333afDhw+rYcOGuuGGG/TVV18pOjpakjRx4kSdPn1ao0aNUk5Ojjp27Ki0tDSFhIQ4t/HCCy/I19dXAwcO1OnTp9W9e3ctXLhQPj4+3josAABQjXg17CxfvvyCj9tsNiUnJys5ObnUPgEBAUpNTVVqamolVwcAAKygWp2zAwAAUNkIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNKq1V3PAbinW7dz//r5SaNGSYmJ5bvLOQBcCpjZAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAllZtws60adNks9mUlJTkbDPGKDk5WZGRkQoMDFTXrl21Y8cOl/Xy8vI0ZswYhYWFKTg4WLfddpsOHjx4kasHAADVVbUIO1u2bNGrr76qq6++2qV9xowZmjVrlubMmaMtW7YoIiJC8fHxOn78uLNPUlKS3n//fS1fvlwbNmzQiRMnlJiYqIKCgot9GAAAoBryetg5ceKEBg0apNdee03169d3thtjNHv2bE2ePFkDBgxQmzZttGjRIp06dUpLly6VJB07dkzz5s3TzJkz1aNHD7Vr106LFy/W999/r9WrV3vrkAAAQDXi6+0CRo8erb59+6pHjx6aOnWqs33Pnj3KyspSQkKCs83f319dunTRxo0bNXLkSGVkZMjhcLj0iYyMVJs2bbRx40b17NmzxH3m5eUpLy/PuZybmytJcjgccjgcbh1H0Xrurg/GsDR+fuXva7c7XP6tqEt96HkNeo4x9BxjWH7lHSOvhp3ly5frm2++0ZYtW4o9lpWVJUkKDw93aQ8PD9e+ffucffz8/FxmhIr6FK1fkmnTpmnKlCnF2tPS0hQUFFTh4/iz9PR0j9YHY3i+UaMqvs6IEe6N4SefuLWa5fAa9Bxj6DnGsGynTp0qVz+vhZ0DBw5o7NixSktLU0BAQKn9bDaby7Ixpljb+crqM2nSJI0bN865nJubq6ioKCUkJKhOnTrlPAJXDodD6enpio+Pl91ud2sblzrGsGSJieXva7c7NGJEul57LV4OR8XH8OOPK7yKpfAa9Bxj6DnGsPyK3pkpi9fCTkZGhrKzs9W+fXtnW0FBgT7//HPNmTNHO3fulHRu9qZx48bOPtnZ2c7ZnoiICOXn5ysnJ8dldic7O1udOnUqdd/+/v7y9/cv1m632z1+YVXGNi51jKGr/PyKr+Nw2JWfX/ExZNjP4TXoOcbQc4xh2co7Pl47Qbl79+76/vvvtXXrVudXhw4dNGjQIG3dulWXX365IiIiXKbx8vPztX79emeQad++vex2u0ufzMxMbd++/YJhBwAAXDq8NrMTEhKiNm3auLQFBwcrNDTU2Z6UlKSUlBTFxsYqNjZWKSkpCgoK0v333y9Jqlu3roYPH67x48crNDRUDRo00IQJExQXF6cePXpc9GMCAADVj9evxrqQiRMn6vTp0xo1apRycnLUsWNHpaWlKSQkxNnnhRdekK+vrwYOHKjTp0+re/fuWrhwoXx8fLxYOQAAqC6qVdhZt26dy7LNZlNycrKSk5NLXScgIECpqalKTU2t2uIAAECN5PUPFQQAAKhKhB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBp1ep2EQCqv27dyu6zdm3V1wEA5cXMDgAAsDTCDgAAsDTCDgAAsDS3ws6ePXsquw4AAIAq4dYJyldccYVuueUWDR8+XHfddZcCAgIquy6gxuNEXgCoHtya2fnuu+/Url07jR8/XhERERo5cqQ2b95c2bUBAAB4zK2w06ZNG82aNUu//vqrFixYoKysLHXu3FlXXXWVZs2apd9//72y6wQAAHCLRyco+/r6qn///nrrrbc0ffp07d69WxMmTFCTJk00ePBgZWZmVladAAAAbvEo7Hz99dcaNWqUGjdurFmzZmnChAnavXu31qxZo19//VW33357ZdUJAADgFrdOUJ41a5YWLFignTt3qk+fPnr99dfVp08f1ap1LjvFxMTolVdeUatWrSq1WAAAgIpyK+zMnTtXw4YN00MPPaSIiIgS+zRt2lTz5s3zqDgAAABPuRV2fvrppzL7+Pn5aciQIe5sHgAAoNK4dc7OggUL9Pbbbxdrf/vtt7Vo0SKPiwIAAKgsboWdZ599VmFhYcXaGzVqpJSUFI+LAgAAqCxuhZ19+/YpJiamWHt0dLT279/vcVEAAACVxa2w06hRI23btq1Y+3fffafQ0FCPiwIAAKgsboWde++9V4899pjWrl2rgoICFRQUaM2aNRo7dqzuvffeyq4RAADAbW5djTV16lTt27dP3bt3l6/vuU0UFhZq8ODBnLMDAACqFbfCjp+fn9588039/e9/13fffafAwEDFxcUpOjq6susDAADwiFthp0iLFi3UokWLyqoFAACg0rkVdgoKCrRw4UJ99tlnys7OVmFhocvja9asqZTiAAAAPOVW2Bk7dqwWLlyovn37qk2bNrLZbJVdFwAAQKVwK+wsX75cb731lvr06VPZ9QAAAFQqty499/Pz0xVXXFHZtQAAAFQ6t8LO+PHj9eKLL8oYU9n1AAAAVCq33sbasGGD1q5dq08//VRXXXWV7Ha7y+PvvfdepRQHAADgKbfCTr169dS/f//KrgUAAKDSuRV2FixYUNl1AAAAVAm3ztmRpLNnz2r16tV65ZVXdPz4cUnSoUOHdOLEiUorDgAAwFNuzezs27dPvXr10v79+5WXl6f4+HiFhIRoxowZOnPmjF5++eXKrhMAAMAtbs3sjB07Vh06dFBOTo4CAwOd7f3799dnn31WacUBAAB4yu2rsb788kv5+fm5tEdHR+vXX3+tlMIAAAAqg1thp7CwUAUFBcXaDx48qJCQEI+LAi4V3bp5uwIAsD633saKj4/X7Nmzncs2m00nTpzQU089xS0kAABAteLWzM4LL7ygbt26qXXr1jpz5ozuv/9+/fTTTwoLC9OyZcsqu0YAAAC3uRV2IiMjtXXrVi1btkzffPONCgsLNXz4cA0aNMjlhGUAAABvcyvsSFJgYKCGDRumYcOGVWY9AAAAlcqtsPP6669f8PHBgweXaztz587V3LlztXfvXknSVVddpb/97W/q3bu3JMkYoylTpujVV19VTk6OOnbsqH/+85+66qqrnNvIy8vThAkTtGzZMp0+fVrdu3fXSy+9pCZNmrhzaAAAwGLcCjtjx451WXY4HDp16pT8/PwUFBRU7rDTpEkTPfvss7riiiskSYsWLdLtt9+ub7/9VldddZVmzJihWbNmaeHChWrRooWmTp2q+Ph47dy503nVV1JSkj766CMtX75coaGhGj9+vBITE5WRkSEfHx93Dg8AAFiIW1dj5eTkuHydOHFCO3fuVOfOnSt0gnK/fv3Up08ftWjRQi1atNAzzzyj2rVr66uvvpIxRrNnz9bkyZM1YMAAtWnTRosWLdKpU6e0dOlSSdKxY8c0b948zZw5Uz169FC7du20ePFiff/991q9erU7hwYAACzG7XtjnS82NlbPPvtssVmf8iooKNDy5ct18uRJ3XjjjdqzZ4+ysrKUkJDg7OPv768uXbpo48aNkqSMjAw5HA6XPpGRkWrTpo2zDwAAuLS5fYJySXx8fHTo0KEKrfP999/rxhtv1JkzZ1S7dm29//77at26tTOshIeHu/QPDw/Xvn37JElZWVny8/NT/fr1i/XJysoqdZ95eXnKy8tzLufm5ko693acw+GoUP1FitZzd31YbwzP+4Dxi8Jud7j8W1HlGfryHFdNfQqt9hr0BsbQc4xh+ZV3jNwKOx9++KHLsjFGmZmZmjNnjm666aYKbatly5baunWr/vjjD7377rsaMmSI1q9f73zcZrMV29f5becrq8+0adM0ZcqUYu1paWkKCgqqUP3nS09P92h9WGcMR43y3r5HjHBvDD/5pOw+5Tmu8mynOrPKa9CbGEPPMYZlO3XqVLn6uRV27rjjDpdlm82mhg0b6tZbb9XMmTMrtC0/Pz/nCcodOnTQli1b9OKLL+qJJ56QdG72pnHjxs7+2dnZztmeiIgI5efnKycnx2V2Jzs7W506dSp1n5MmTdK4ceOcy7m5uYqKilJCQoLq1KlTofqLOBwOpaenKz4+Xna73a1tXOqsNoaJiRd/n3a7QyNGpOu11+LlcHhvDD/+2Gu79ojVXoPewBh6jjEsv6J3Zsri9r2xqooxRnl5eYqJiVFERITS09PVrl07SVJ+fr7Wr1+v6dOnS5Lat28vu92u9PR0DRw4UJKUmZmp7du3a8aMGaXuw9/fX/7+/sXa7Xa7xy+sytjGpc4qY5if7719Oxx25ed7bwxr+tNnldegNzGGnmMMy1be8anUc3Yq6sknn1Tv3r0VFRWl48ePa/ny5Vq3bp1Wrlwpm82mpKQkpaSkKDY2VrGxsUpJSVFQUJDuv/9+SVLdunU1fPhwjR8/XqGhoWrQoIEmTJiguLg49ejRw5uHBgAAqgm3ws6f3wIqy6xZs0p97LffftODDz6ozMxM1a1bV1dffbVWrlyp+Ph4SdLEiRN1+vRpjRo1yvmhgmlpaS53Vn/hhRfk6+urgQMHOj9UcOHChXzGDgAAkORm2Pn222/1zTff6OzZs2rZsqUkadeuXfLx8dG1117r7FfWicTz5s274OM2m03JyclKTk4utU9AQIBSU1OVmppa/gMAAACXDLfCTr9+/RQSEqJFixY5TwzOycnRQw89pJtvvlnjx4+v1CIBAADc5daHCs6cOVPTpk1zuQKqfv36mjp1aoWvxgIAAKhKboWd3Nxc/fbbb8Xas7Ozdfz4cY+LAgAAqCxuhZ3+/fvroYce0jvvvKODBw/q4MGDeueddzR8+HANGDCgsmsEAABwm1vn7Lz88suaMGGCHnjgAedHNfv6+mr48OF67rnnKrVAAAAAT7gVdoKCgvTSSy/pueee0+7du2WM0RVXXKHg4ODKrg8AAMAjHt31PDMzU5mZmWrRooWCg4NljKmsugAAACqFW2HnyJEj6t69u1q0aKE+ffooMzNTkvTwww9z2TkAAKhW3Ao7jz/+uOx2u/bv3+9yl/B77rlHK1eurLTiAAAAPOXWOTtpaWlatWqVmjRp4tIeGxurffv2VUphAAAAlcGtmZ2TJ0+6zOgUOXz4cIl3EwcAAPAWt8LOLbfcotdff925bLPZVFhYqOeee07dunWrtOIAAAA85dbbWM8995y6du2qr7/+Wvn5+Zo4caJ27Niho0eP6ssvv6zsGgEAANzm1sxO69attW3bNl1//fWKj4/XyZMnNWDAAH377bdq3rx5ZdcIAADgtgrP7DgcDiUkJOiVV17RlClTqqImAACASlPhmR273a7t27fLZrNVRT0AAACVyq23sQYPHqx58+ZVdi0AAACVzq0TlPPz8/Wvf/1L6enp6tChQ7F7Ys2aNatSigMAAPBUhcLOL7/8ombNmmn79u269tprJUm7du1y6cPbWwAAoDqpUNiJjY1VZmam1q5dK+nc7SH+8Y9/KDw8vEqKAwAA8FSFztk5/67mn376qU6ePFmpBQEAAFQmt05QLnJ++AEAAKhuKhR2bDZbsXNyOEcHAABUZxU6Z8cYo6FDhzpv9nnmzBk98sgjxa7Geu+99yqvQgAAAA9UKOwMGTLEZfmBBx6o1GIAAAAqW4XCzoIFC6qqDgAAgCrh0QnKAAAA1R1hBwAAWBphBwAAWBphBwAAWBphBwAAWBphBwAAWBphBwAAWBphBwAAWBphBwAAWBphBwAAWBphBwAAWBphBwAAWBphBwAAWFqF7noOAOXRrVvZfdaurfo6AEBiZgcAAFgcYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFiaV8POtGnTdN111ykkJESNGjXSHXfcoZ07d7r0McYoOTlZkZGRCgwMVNeuXbVjxw6XPnl5eRozZozCwsIUHBys2267TQcPHryYhwIAAKopr94uYv369Ro9erSuu+46nT17VpMnT1ZCQoJ++OEHBQcHS5JmzJihWbNmaeHChWrRooWmTp2q+Ph47dy5UyEhIZKkpKQkffTRR1q+fLlCQ0M1fvx4JSYmKiMjQz4+Pt48RFhUeW6HAACoHrwadlauXOmyvGDBAjVq1EgZGRm65ZZbZIzR7NmzNXnyZA0YMECStGjRIoWHh2vp0qUaOXKkjh07pnnz5umNN95Qjx49JEmLFy9WVFSUVq9erZ49e1704wIAANVHtboR6LFjxyRJDRo0kCTt2bNHWVlZSkhIcPbx9/dXly5dtHHjRo0cOVIZGRlyOBwufSIjI9WmTRtt3LixxLCTl5envLw853Jubq4kyeFwyOFwuFV70Xruro+aNYZ+ft6uoGR2u8Pl3+qsOj7NNek1WF0xhp5jDMuvvGNUbcKOMUbjxo1T586d1aZNG0lSVlaWJCk8PNylb3h4uPbt2+fs4+fnp/r16xfrU7T++aZNm6YpU6YUa09LS1NQUJBHx5Genu7R+qgZYzhqlLcruLARI6r/GH7yibcrKF1NeA1Wd4yh5xjDsp06dapc/apN2Hn00Ue1bds2bdiwodhjNpvNZdkYU6ztfBfqM2nSJI0bN865nJubq6ioKCUkJKhOnTpuVH8uXaanpys+Pl52u92tbVzqatIYJiZ6u4KS2e0OjRiRrtdei5fDUb3H8OOPvV1BcTXpNVhdMYaeYwzLr+idmbJUi7AzZswYffjhh/r888/VpEkTZ3tERISkc7M3jRs3drZnZ2c7Z3siIiKUn5+vnJwcl9md7OxsderUqcT9+fv7y9/fv1i73W73+IVVGdu41NWEMczP93YFF+Zw2JWfX73HsDo/xTXhNVjdMYaeYwzLVt7x8eql58YYPfroo3rvvfe0Zs0axcTEuDweExOjiIgIl6m8/Px8rV+/3hlk2rdvL7vd7tInMzNT27dvLzXsAACAS4dXZ3ZGjx6tpUuX6oMPPlBISIjzHJu6desqMDBQNptNSUlJSklJUWxsrGJjY5WSkqKgoCDdf//9zr7Dhw/X+PHjFRoaqgYNGmjChAmKi4tzXp0FAAAuXV4NO3PnzpUkde3a1aV9wYIFGjp0qCRp4sSJOn36tEaNGqWcnBx17NhRaWlpzs/YkaQXXnhBvr6+GjhwoE6fPq3u3btr4cKFfMYOAADwbtgxxpTZx2azKTk5WcnJyaX2CQgIUGpqqlJTUyuxOgAAYAXcGwsAAFgaYQcAAFgaYQcAAFhatficHaA64SafAGAtzOwAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABL49JzAF5Rnkv8166t+joAWB8zOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNK8GnY+//xz9evXT5GRkbLZbFqxYoXL48YYJScnKzIyUoGBgeratat27Njh0icvL09jxoxRWFiYgoODddttt+ngwYMX8SgAAEB15tWwc/LkSV1zzTWaM2dOiY/PmDFDs2bN0pw5c7RlyxZFREQoPj5ex48fd/ZJSkrS+++/r+XLl2vDhg06ceKEEhMTVVBQcLEOAwAAVGO+3tx579691bt37xIfM8Zo9uzZmjx5sgYMGCBJWrRokcLDw7V06VKNHDlSx44d07x58/TGG2+oR48ekqTFixcrKipKq1evVs+ePS/asQAAgOrJq2HnQvbs2aOsrCwlJCQ42/z9/dWlSxdt3LhRI0eOVEZGhhwOh0ufyMhItWnTRhs3biw17OTl5SkvL8+5nJubK0lyOBxyOBxu1Vu0nrvro/qMoZ+fV3fvEbvd4fJvTXexXwrV5TVYkzGGnmMMy6+8Y1Rtw05WVpYkKTw83KU9PDxc+/btc/bx8/NT/fr1i/UpWr8k06ZN05QpU4q1p6WlKSgoyKO609PTPVof3h/DUaO8uvtKMWKENV6Hn3zinf16+zVoBYyh5xjDsp06dapc/apt2Clis9lclo0xxdrOV1afSZMmady4cc7l3NxcRUVFKSEhQXXq1HGrTofDofT0dMXHx8tut7u1jUtddRnDxESv7dpjdrtDI0ak67XX4uVw1PzX4ccfX9z9VZfXYE3GGHqOMSy/ondmylJtw05ERISkc7M3jRs3drZnZ2c7Z3siIiKUn5+vnJwcl9md7OxsderUqdRt+/v7y9/fv1i73W73+IVVGdu41Hl7DPPzvbbrSuNw2JWfX/Nfh956GXj7NWgFjKHnGMOylXd8qu3n7MTExCgiIsJlGi8/P1/r1693Bpn27dvLbre79MnMzNT27dsvGHYAAMClw6szOydOnNDPP//sXN6zZ4+2bt2qBg0aqGnTpkpKSlJKSopiY2MVGxurlJQUBQUF6f7775ck1a1bV8OHD9f48eMVGhqqBg0aaMKECYqLi3NenQUAAC5tXg07X3/9tbp16+ZcLjqPZsiQIVq4cKEmTpyo06dPa9SoUcrJyVHHjh2VlpamkJAQ5zovvPCCfH19NXDgQJ0+fVrdu3fXwoUL5ePjc9GPBwAAVD9eDTtdu3aVMabUx202m5KTk5WcnFxqn4CAAKWmpio1NbUKKgQAADVdtT1nBwAAoDIQdgAAgKURdgAAgKURdgAAgKURdgAAgKURdgAAgKVV29tFAMCfPoarVGvXVn0dAGo2ZnYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAICl8aGCuKSU50PqAADWwswOAACwNMIOAACwNMIOAACwNM7ZAVCjcbNQAGVhZgcAAFgaYQcAAFgaYQcAAFga5+zAMvgMHQBASZjZAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlkbYAQAAlsZdzwFYXrduZfdZu7bq6wDgHczsAAAAS2NmBzVCef4yBwCgJMzsAAAASyPsAAAASyPsAAAAS+OcHXhd0fk4fn7SqFFSYqKUn+/dmnDp6dat7NcgV2wBNRMzOwAAwNIIOwAAwNIIOwAAwNI4Zwdu41NpcanhNQ/UTJaZ2XnppZcUExOjgIAAtW/fXl988YW3SwIAANWAJcLOm2++qaSkJE2ePFnffvutbr75ZvXu3Vv79+/3dmkAAMDLLPE21qxZszR8+HA9/PDDkqTZs2dr1apVmjt3rqZNm+bl6i5t3OYBKI63w4CLq8aHnfz8fGVkZOi///u/XdoTEhK0ceNGL1UF4FJFwAeqnxofdg4fPqyCggKFh4e7tIeHhysrK6vEdfLy8pSXl+dcPnbsmCTp6NGjcjgcbtXhcDh06tQpHTlyRHa73dk+cGDZ6771llu7rFLlqbtWpb8Jem4MpSOqVcteZm+UhDH0TPUZv+7dy+5zMX92lPdnWWk/C71VT010scawsnjzuTh+/LgkyRhzwX41PuwUsdlsLsvGmGJtRaZNm6YpU6YUa4+JiamS2soSFuaV3VZLaWnerqDmYww9U5PGr7r97KAelKaqn4vjx4+rbt26pT5e48NOWFiYfHx8is3iZGdnF5vtKTJp0iSNGzfOuVxYWKijR48qNDS01IBUltzcXEVFRenAgQOqU6eOW9u41DGGnmMMPcP4eY4x9BxjWH7GGB0/flyRkZEX7Ffjw46fn5/at2+v9PR09e/f39menp6u22+/vcR1/P395e/v79JWr169SqmnTp06vDg9xBh6jjH0DOPnOcbQc4xh+VxoRqdIjQ87kjRu3Dg9+OCD6tChg2688Ua9+uqr2r9/vx555BFvlwYAALzMEmHnnnvu0ZEjR/T0008rMzNTbdq00SeffKLo6GhvlwYAALzMEmFHkkaNGqVRo0Z5bf/+/v566qmnir09hvJjDD3HGHqG8fMcY+g5xrDy2UxZ12sBAADUYJa4XQQAAEBpCDsAAMDSCDsAAMDSCDsAAMDSCDtVLC8vT23btpXNZtPWrVu9XU6NsHfvXg0fPlwxMTEKDAxU8+bN9dRTTyk/P9/bpVVrL730kmJiYhQQEKD27dvriy++8HZJNca0adN03XXXKSQkRI0aNdIdd9yhnTt3erusGmvatGmy2WxKSkrydik1yq+//qoHHnhAoaGhCgoKUtu2bZWRkeHtsiyBsFPFJk6cWObHWMPVv//9bxUWFuqVV17Rjh079MILL+jll1/Wk08+6e3Sqq0333xTSUlJmjx5sr799lvdfPPN6t27t/bv3+/t0mqE9evXa/To0frqq6+Unp6us2fPKiEhQSdPnvR2aTXOli1b9Oqrr+rqq6/2dik1Sk5Ojm666SbZ7XZ9+umn+uGHHzRz5sxK+3T/S55Blfnkk09Mq1atzI4dO4wk8+2333q7pBprxowZJiYmxttlVFvXX3+9eeSRR1zaWrVqZf77v//bSxXVbNnZ2UaSWb9+vbdLqVGOHz9uYmNjTXp6uunSpYsZO3ast0uqMZ544gnTuXNnb5dhWczsVJHffvtNI0aM0BtvvKGgoCBvl1PjHTt2TA0aNPB2GdVSfn6+MjIylJCQ4NKekJCgjRs3eqmqmu3YsWOSxGuugkaPHq2+ffuqR48e3i6lxvnwww/VoUMH3X333WrUqJHatWun1157zdtlWQZhpwoYYzR06FA98sgj6tChg7fLqfF2796t1NRU7nVWisOHD6ugoEDh4eEu7eHh4crKyvJSVTWXMUbjxo1T586d1aZNG2+XU2MsX75c33zzjaZNm+btUmqkX375RXPnzlVsbKxWrVqlRx55RI899phef/11b5dmCYSdCkhOTpbNZrvg19dff63U1FTl5uZq0qRJ3i65Winv+P3ZoUOH1KtXL9199916+OGHvVR5zWCz2VyWjTHF2lC2Rx99VNu2bdOyZcu8XUqNceDAAY0dO1aLFy9WQECAt8upkQoLC3XttdcqJSVF7dq108iRIzVixAjNnTvX26VZgmXujXUxPProo7r33nsv2KdZs2aaOnWqvvrqq2L3NenQoYMGDRqkRYsWVWWZ1VZ5x6/IoUOH1K1bN+ed7FGysLAw+fj4FJvFyc7OLjbbgwsbM2aMPvzwQ33++edq0qSJt8upMTIyMpSdna327ds72woKCvT5559rzpw5ysvLk4+PjxcrrP4aN26s1q1bu7RdeeWVevfdd71UkbUQdiogLCxMYWFhZfb7xz/+oalTpzqXDx06pJ49e+rNN99Ux44dq7LEaq284yeduwSzW7duat++vRYsWKBatZiELI2fn5/at2+v9PR09e/f39menp6u22+/3YuV1RzGGI0ZM0bvv/++1q1bp5iYGG+XVKN0795d33//vUvbQw89pFatWumJJ54g6JTDTTfdVOzjDnbt2qXo6GgvVWQthJ0q0LRpU5fl2rVrS5KaN2/OX4vlcOjQIXXt2lVNmzbV888/r99//935WEREhBcrq77GjRunBx98UB06dHDOhO3fv5/znMpp9OjRWrp0qT744AOFhIQ4Z8nq1q2rwMBAL1dX/YWEhBQ7vyk4OFihoaGc91ROjz/+uDp16qSUlBQNHDhQmzdv1quvvsqsdiUh7KDaSUtL088//6yff/65WDg0xnipqurtnnvu0ZEjR/T0008rMzNTbdq00SeffMJfheVUdF5E165dXdoXLFigoUOHXvyCcMm57rrr9P7772vSpEl6+umnFRMTo9mzZ2vQoEHeLs0SbIbfHgAAwMI4EQIAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQeAV9lsNq1YscK5/O9//1s33HCDAgIC1LZt21LbAHjH559/rn79+ikyMrLY929VaNasWYk3jh49enS5t0HYAVAlhg4d6vyhZLfbFR4ervj4eM2fP1+FhYXOfpmZmerdu7dz+amnnlJwcLB27typzz77rNQ2AN5x8uRJXXPNNZozZ85F2d+WLVuUmZnp/EpPT5ck3X333eXeBmEHQJXp1auXMjMztXfvXn366afq1q2bxo4dq8TERJ09e1bSufud+fv7O9fZvXu3OnfurOjoaIWGhpbaVlH5+fmeHxAA9e7dW1OnTtWAAQNKfDw/P18TJ07UZZddpuDgYHXs2FHr1q1ze38NGzZURESE8+vjjz9W8+bN1aVLl3Jvg7ADoMr4+/srIiJCl112ma699lo9+eST+uCDD/Tpp59q4cKFklzfxrLZbMrIyNDTTz8tm82m5OTkEtsk6ddff9U999yj+vXrKzQ0VLfffrv27t3r3PfQoUN1xx13aNq0aYqMjFSLFi0qtN7zzz+vxo0bKzQ0VKNHj5bD4XD2ycvL08SJExUVFSV/f3/FxsZq3rx5zsd/+OEH9enTR7Vr11Z4eLgefPBBHT58uErGGKhuHnroIX355Zdavny5tm3bprvvvlu9evXSTz/95PG28/PztXjxYg0bNkw2m63c6xF2AFxUt956q6655hq99957xR7LzMzUVVddpfHjxyszM1MTJkwose3UqVPq1q2bateurc8//1wbNmxQ7dq11atXL5cZnM8++0w//vij0tPT9fHHH5d7vbVr12r37t1au3atFi1apIULFzrDmSQNHjxYy5cv1z/+8Q/9+OOPevnll1W7dm3nMXTp0kVt27bV119/rZUrV+q3337TwIEDq25QgWpi9+7dWrZsmd5++23dfPPNat68uSZMmKDOnTtrwYIFHm9/xYoV+uOPPyp8g17ueg7gomvVqpW2bdtWrD0iIkK+vr6qXbu2IiIiJEm1a9cu1jZ//nzVqlVL//rXv5x/3S1YsED16tXTunXrlJCQIEkKDg7Wv/71L/n5+VVovfr162vOnDny8fFRq1at1LdvX3322WcaMWKEdu3apbfeekvp6enq0aOHJOnyyy93HsPcuXN17bXXKiUlxdk2f/58RUVFadeuXc4ZJsCKvvnmGxljir3O8/LynG9B7927VzExMRfczujRo0s8J2jevHnq3bu3IiMjK1QXYQfARWeMqdAU9PkyMjL0888/KyQkxKX9zJkz2r17t3M5Li7OGXQqst5VV10lHx8f53Ljxo31/fffS5K2bt0qHx+fUs8XyMjI0Nq1a50zPX+2e/duwg4srbCwUD4+PsrIyHD5HpLk/J647LLL9OOPP15wO/Xr1y/Wtm/fPq1evbrEWeGyEHYAXHQ//vhjmX/ZXUhhYaHat2+vJUuWFHusYcOGzv8HBwe7tZ7dbnd5zGazOa8gCwwMLLO2fv36afr06cUea9y48QXXBWq6du3aqaCgQNnZ2br55ptL7GO329WqVasKb3vBggVq1KiR+vbtW+F1CTsALqo1a9bo+++/1+OPP+72Nq699lq9+eabatSokerUqVPl6/1ZXFycCgsLtX79eufbWOfv491331WzZs3k68uPWFjPiRMn9PPPPzuX9+zZo61bt6pBgwZq0aKFBg0apMGDB2vmzJlq166dDh8+rDVr1iguLk59+vRxa5+FhYVasGCBhgwZ4tb3FScoA6gyeXl5ysrK0q+//qpvvvlGKSkpuv3225WYmKjBgwe7vd1BgwYpLCxMt99+u7744gvt2bNH69ev19ixY3Xw4MFKX+/PmjVrpiFDhmjYsGFasWKF9uzZo3Xr1umtt96SdO5cg6NHj+q+++7T5s2b9csvvygtLU3Dhg1TQUGB28cMVBdff/212rVrp3bt2kmSxo0bp3bt2ulvf/ubpHMzMIMHD9b48ePVsmVL3Xbbbfq///s/RUVFub3P1atXa//+/Ro2bJhb6/NnB4Aqs3LlSjVu3Fi+vr6qX7++rrnmGv3jH//QkCFDVKuW+39rBQUF6fPPP9cTTzyhAQMG6Pjx47rsssvUvXv3C87YuLve+ebOnasnn3xSo0aN0pEjR9S0aVM9+eSTkqTIyEh9+eWXeuKJJ9SzZ0/l5eUpOjpavXr18uiYgeqia9euMsaU+rjdbteUKVM0ZcqUSttnQkLCBfdZFpvxZG0AAIBqjj8zAACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApRF2AACApf1/NuhuLZJw5e0AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "differences = tl_model.blocks[0].attn.forward(test_tensor, test_tensor, test_tensor) - hf_model.model.layers[0].self_attn.forward(test_tensor)[0]\n",
    "\n",
    "# Flatten the differences to create a one-dimensional tensor\n",
    "flattened_differences = differences.flatten().cpu().detach().numpy()\n",
    "\n",
    "# Plot the histogram of the differences\n",
    "plt.hist(flattened_differences, bins=50, alpha=0.75, color='blue')\n",
    "plt.title('Differences Between Attention Outputs')\n",
    "plt.xlabel('Difference')\n",
    "plt.ylabel('Frequency')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
